1a246b010SChuck Lever /* 2a246b010SChuck Lever * linux/net/sunrpc/xprtsock.c 3a246b010SChuck Lever * 4a246b010SChuck Lever * Client-side transport implementation for sockets. 5a246b010SChuck Lever * 6113aa838SAlan Cox * TCP callback races fixes (C) 1998 Red Hat 7113aa838SAlan Cox * TCP send fixes (C) 1998 Red Hat 8a246b010SChuck Lever * TCP NFS related read + write fixes 9a246b010SChuck Lever * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie> 10a246b010SChuck Lever * 11a246b010SChuck Lever * Rewrite of larges part of the code in order to stabilize TCP stuff. 12a246b010SChuck Lever * Fix behaviour when socket buffer is full. 13a246b010SChuck Lever * (C) 1999 Trond Myklebust <trond.myklebust@fys.uio.no> 1455aa4f58SChuck Lever * 1555aa4f58SChuck Lever * IP socket transport implementation, (C) 2005 Chuck Lever <cel@netapp.com> 168f9d5b1aSChuck Lever * 178f9d5b1aSChuck Lever * IPv6 support contributed by Gilles Quillard, Bull Open Source, 2005. 188f9d5b1aSChuck Lever * <gilles.quillard@bull.net> 19a246b010SChuck Lever */ 20a246b010SChuck Lever 21a246b010SChuck Lever #include <linux/types.h> 22176e21eeSChuck Lever #include <linux/string.h> 23a246b010SChuck Lever #include <linux/slab.h> 24bc25571eS\"Talpey, Thomas\ #include <linux/module.h> 25a246b010SChuck Lever #include <linux/capability.h> 26a246b010SChuck Lever #include <linux/pagemap.h> 27a246b010SChuck Lever #include <linux/errno.h> 28a246b010SChuck Lever #include <linux/socket.h> 29a246b010SChuck Lever #include <linux/in.h> 30a246b010SChuck Lever #include <linux/net.h> 31a246b010SChuck Lever #include <linux/mm.h> 32176e21eeSChuck Lever #include <linux/un.h> 33a246b010SChuck Lever #include <linux/udp.h> 34a246b010SChuck Lever #include <linux/tcp.h> 35a246b010SChuck Lever #include <linux/sunrpc/clnt.h> 365976687aSJeff Layton #include <linux/sunrpc/addr.h> 3702107148SChuck Lever #include <linux/sunrpc/sched.h> 384cfc7e60SRahul Iyer #include <linux/sunrpc/svcsock.h> 3949c36fccS\"Talpey, Thomas\ #include <linux/sunrpc/xprtsock.h> 40a246b010SChuck Lever #include <linux/file.h> 419e00abc3STrond Myklebust #ifdef CONFIG_SUNRPC_BACKCHANNEL 4244b98efdSRicardo Labiaga #include <linux/sunrpc/bc_xprt.h> 4344b98efdSRicardo Labiaga #endif 44a246b010SChuck Lever 45a246b010SChuck Lever #include <net/sock.h> 46a246b010SChuck Lever #include <net/checksum.h> 47a246b010SChuck Lever #include <net/udp.h> 48a246b010SChuck Lever #include <net/tcp.h> 49a246b010SChuck Lever 5040b5ea0cSTrond Myklebust #include <trace/events/sunrpc.h> 5140b5ea0cSTrond Myklebust 524cfc7e60SRahul Iyer #include "sunrpc.h" 53176e21eeSChuck Lever 54176e21eeSChuck Lever static void xs_close(struct rpc_xprt *xprt); 55176e21eeSChuck Lever 569903cd1cSChuck Lever /* 57c556b754SChuck Lever * xprtsock tunables 58c556b754SChuck Lever */ 5909acfea5STrond Myklebust static unsigned int xprt_udp_slot_table_entries = RPC_DEF_SLOT_TABLE; 6009acfea5STrond Myklebust static unsigned int xprt_tcp_slot_table_entries = RPC_MIN_SLOT_TABLE; 6109acfea5STrond Myklebust static unsigned int xprt_max_tcp_slot_table_entries = RPC_MAX_SLOT_TABLE; 62c556b754SChuck Lever 6309acfea5STrond Myklebust static unsigned int xprt_min_resvport = RPC_DEF_MIN_RESVPORT; 6409acfea5STrond Myklebust static unsigned int xprt_max_resvport = RPC_DEF_MAX_RESVPORT; 65c556b754SChuck Lever 6654c09874STrond Myklebust #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) 6754c09874STrond Myklebust 687d1e8255STrond Myklebust #define XS_TCP_LINGER_TO (15U * HZ) 6925fe6142STrond Myklebust static unsigned int xs_tcp_fin_timeout __read_mostly = XS_TCP_LINGER_TO; 707d1e8255STrond Myklebust 71c556b754SChuck Lever /* 72fbf76683SChuck Lever * We can register our own files under /proc/sys/sunrpc by 73fbf76683SChuck Lever * calling register_sysctl_table() again. The files in that 74fbf76683SChuck Lever * directory become the union of all files registered there. 75fbf76683SChuck Lever * 76fbf76683SChuck Lever * We simply need to make sure that we don't collide with 77fbf76683SChuck Lever * someone else's file names! 78fbf76683SChuck Lever */ 79fbf76683SChuck Lever 80fbf76683SChuck Lever static unsigned int min_slot_table_size = RPC_MIN_SLOT_TABLE; 81fbf76683SChuck Lever static unsigned int max_slot_table_size = RPC_MAX_SLOT_TABLE; 82d9ba131dSTrond Myklebust static unsigned int max_tcp_slot_table_limit = RPC_MAX_SLOT_TABLE_LIMIT; 83fbf76683SChuck Lever static unsigned int xprt_min_resvport_limit = RPC_MIN_RESVPORT; 84fbf76683SChuck Lever static unsigned int xprt_max_resvport_limit = RPC_MAX_RESVPORT; 85fbf76683SChuck Lever 86fbf76683SChuck Lever static struct ctl_table_header *sunrpc_table_header; 87fbf76683SChuck Lever 88fbf76683SChuck Lever /* 89fbf76683SChuck Lever * FIXME: changing the UDP slot table size should also resize the UDP 90fbf76683SChuck Lever * socket buffers for existing UDP transports 91fbf76683SChuck Lever */ 92fe2c6338SJoe Perches static struct ctl_table xs_tunables_table[] = { 93fbf76683SChuck Lever { 94fbf76683SChuck Lever .procname = "udp_slot_table_entries", 95fbf76683SChuck Lever .data = &xprt_udp_slot_table_entries, 96fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 97fbf76683SChuck Lever .mode = 0644, 986d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 99fbf76683SChuck Lever .extra1 = &min_slot_table_size, 100fbf76683SChuck Lever .extra2 = &max_slot_table_size 101fbf76683SChuck Lever }, 102fbf76683SChuck Lever { 103fbf76683SChuck Lever .procname = "tcp_slot_table_entries", 104fbf76683SChuck Lever .data = &xprt_tcp_slot_table_entries, 105fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 106fbf76683SChuck Lever .mode = 0644, 1076d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 108fbf76683SChuck Lever .extra1 = &min_slot_table_size, 109fbf76683SChuck Lever .extra2 = &max_slot_table_size 110fbf76683SChuck Lever }, 111fbf76683SChuck Lever { 112d9ba131dSTrond Myklebust .procname = "tcp_max_slot_table_entries", 113d9ba131dSTrond Myklebust .data = &xprt_max_tcp_slot_table_entries, 114d9ba131dSTrond Myklebust .maxlen = sizeof(unsigned int), 115d9ba131dSTrond Myklebust .mode = 0644, 116d9ba131dSTrond Myklebust .proc_handler = proc_dointvec_minmax, 117d9ba131dSTrond Myklebust .extra1 = &min_slot_table_size, 118d9ba131dSTrond Myklebust .extra2 = &max_tcp_slot_table_limit 119d9ba131dSTrond Myklebust }, 120d9ba131dSTrond Myklebust { 121fbf76683SChuck Lever .procname = "min_resvport", 122fbf76683SChuck Lever .data = &xprt_min_resvport, 123fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 124fbf76683SChuck Lever .mode = 0644, 1256d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 126fbf76683SChuck Lever .extra1 = &xprt_min_resvport_limit, 127fbf76683SChuck Lever .extra2 = &xprt_max_resvport_limit 128fbf76683SChuck Lever }, 129fbf76683SChuck Lever { 130fbf76683SChuck Lever .procname = "max_resvport", 131fbf76683SChuck Lever .data = &xprt_max_resvport, 132fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 133fbf76683SChuck Lever .mode = 0644, 1346d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 135fbf76683SChuck Lever .extra1 = &xprt_min_resvport_limit, 136fbf76683SChuck Lever .extra2 = &xprt_max_resvport_limit 137fbf76683SChuck Lever }, 138fbf76683SChuck Lever { 13925fe6142STrond Myklebust .procname = "tcp_fin_timeout", 14025fe6142STrond Myklebust .data = &xs_tcp_fin_timeout, 14125fe6142STrond Myklebust .maxlen = sizeof(xs_tcp_fin_timeout), 14225fe6142STrond Myklebust .mode = 0644, 1436d456111SEric W. Biederman .proc_handler = proc_dointvec_jiffies, 14425fe6142STrond Myklebust }, 145f8572d8fSEric W. Biederman { }, 146fbf76683SChuck Lever }; 147fbf76683SChuck Lever 148fe2c6338SJoe Perches static struct ctl_table sunrpc_table[] = { 149fbf76683SChuck Lever { 150fbf76683SChuck Lever .procname = "sunrpc", 151fbf76683SChuck Lever .mode = 0555, 152fbf76683SChuck Lever .child = xs_tunables_table 153fbf76683SChuck Lever }, 154f8572d8fSEric W. Biederman { }, 155fbf76683SChuck Lever }; 156fbf76683SChuck Lever 157fbf76683SChuck Lever #endif 158fbf76683SChuck Lever 159fbf76683SChuck Lever /* 16003bf4b70SChuck Lever * Wait duration for a reply from the RPC portmapper. 16103bf4b70SChuck Lever */ 16203bf4b70SChuck Lever #define XS_BIND_TO (60U * HZ) 16303bf4b70SChuck Lever 16403bf4b70SChuck Lever /* 16503bf4b70SChuck Lever * Delay if a UDP socket connect error occurs. This is most likely some 16603bf4b70SChuck Lever * kind of resource problem on the local host. 16703bf4b70SChuck Lever */ 16803bf4b70SChuck Lever #define XS_UDP_REEST_TO (2U * HZ) 16903bf4b70SChuck Lever 17003bf4b70SChuck Lever /* 17103bf4b70SChuck Lever * The reestablish timeout allows clients to delay for a bit before attempting 17203bf4b70SChuck Lever * to reconnect to a server that just dropped our connection. 17303bf4b70SChuck Lever * 17403bf4b70SChuck Lever * We implement an exponential backoff when trying to reestablish a TCP 17503bf4b70SChuck Lever * transport connection with the server. Some servers like to drop a TCP 17603bf4b70SChuck Lever * connection when they are overworked, so we start with a short timeout and 17703bf4b70SChuck Lever * increase over time if the server is down or not responding. 17803bf4b70SChuck Lever */ 17903bf4b70SChuck Lever #define XS_TCP_INIT_REEST_TO (3U * HZ) 18003bf4b70SChuck Lever #define XS_TCP_MAX_REEST_TO (5U * 60 * HZ) 18103bf4b70SChuck Lever 18203bf4b70SChuck Lever /* 18303bf4b70SChuck Lever * TCP idle timeout; client drops the transport socket if it is idle 18403bf4b70SChuck Lever * for this long. Note that we also timeout UDP sockets to prevent 18503bf4b70SChuck Lever * holding port numbers when there is no RPC traffic. 18603bf4b70SChuck Lever */ 18703bf4b70SChuck Lever #define XS_IDLE_DISC_TO (5U * 60 * HZ) 18803bf4b70SChuck Lever 189f895b252SJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) 190a246b010SChuck Lever # undef RPC_DEBUG_DATA 1919903cd1cSChuck Lever # define RPCDBG_FACILITY RPCDBG_TRANS 192a246b010SChuck Lever #endif 193a246b010SChuck Lever 194a246b010SChuck Lever #ifdef RPC_DEBUG_DATA 1959903cd1cSChuck Lever static void xs_pktdump(char *msg, u32 *packet, unsigned int count) 196a246b010SChuck Lever { 197a246b010SChuck Lever u8 *buf = (u8 *) packet; 198a246b010SChuck Lever int j; 199a246b010SChuck Lever 200a246b010SChuck Lever dprintk("RPC: %s\n", msg); 201a246b010SChuck Lever for (j = 0; j < count && j < 128; j += 4) { 202a246b010SChuck Lever if (!(j & 31)) { 203a246b010SChuck Lever if (j) 204a246b010SChuck Lever dprintk("\n"); 205a246b010SChuck Lever dprintk("0x%04x ", j); 206a246b010SChuck Lever } 207a246b010SChuck Lever dprintk("%02x%02x%02x%02x ", 208a246b010SChuck Lever buf[j], buf[j+1], buf[j+2], buf[j+3]); 209a246b010SChuck Lever } 210a246b010SChuck Lever dprintk("\n"); 211a246b010SChuck Lever } 212a246b010SChuck Lever #else 2139903cd1cSChuck Lever static inline void xs_pktdump(char *msg, u32 *packet, unsigned int count) 214a246b010SChuck Lever { 215a246b010SChuck Lever /* NOP */ 216a246b010SChuck Lever } 217a246b010SChuck Lever #endif 218a246b010SChuck Lever 2192118071dSTrond Myklebust static inline struct rpc_xprt *xprt_from_sock(struct sock *sk) 2202118071dSTrond Myklebust { 2212118071dSTrond Myklebust return (struct rpc_xprt *) sk->sk_user_data; 2222118071dSTrond Myklebust } 2232118071dSTrond Myklebust 22495392c59SChuck Lever static inline struct sockaddr *xs_addr(struct rpc_xprt *xprt) 225edb267a6SChuck Lever { 22695392c59SChuck Lever return (struct sockaddr *) &xprt->addr; 22795392c59SChuck Lever } 22895392c59SChuck Lever 229176e21eeSChuck Lever static inline struct sockaddr_un *xs_addr_un(struct rpc_xprt *xprt) 230176e21eeSChuck Lever { 231176e21eeSChuck Lever return (struct sockaddr_un *) &xprt->addr; 232176e21eeSChuck Lever } 233176e21eeSChuck Lever 23495392c59SChuck Lever static inline struct sockaddr_in *xs_addr_in(struct rpc_xprt *xprt) 23595392c59SChuck Lever { 23695392c59SChuck Lever return (struct sockaddr_in *) &xprt->addr; 23795392c59SChuck Lever } 23895392c59SChuck Lever 23995392c59SChuck Lever static inline struct sockaddr_in6 *xs_addr_in6(struct rpc_xprt *xprt) 24095392c59SChuck Lever { 24195392c59SChuck Lever return (struct sockaddr_in6 *) &xprt->addr; 24295392c59SChuck Lever } 24395392c59SChuck Lever 244c877b849SChuck Lever static void xs_format_common_peer_addresses(struct rpc_xprt *xprt) 245c877b849SChuck Lever { 246c877b849SChuck Lever struct sockaddr *sap = xs_addr(xprt); 2479dc3b095SChuck Lever struct sockaddr_in6 *sin6; 2489dc3b095SChuck Lever struct sockaddr_in *sin; 249176e21eeSChuck Lever struct sockaddr_un *sun; 250c877b849SChuck Lever char buf[128]; 251c877b849SChuck Lever 2529dc3b095SChuck Lever switch (sap->sa_family) { 253176e21eeSChuck Lever case AF_LOCAL: 254176e21eeSChuck Lever sun = xs_addr_un(xprt); 255176e21eeSChuck Lever strlcpy(buf, sun->sun_path, sizeof(buf)); 256176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] = 257176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL); 258176e21eeSChuck Lever break; 2599dc3b095SChuck Lever case AF_INET: 260176e21eeSChuck Lever (void)rpc_ntop(sap, buf, sizeof(buf)); 261176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] = 262176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL); 2639dc3b095SChuck Lever sin = xs_addr_in(xprt); 264fc0b5791SJoe Perches snprintf(buf, sizeof(buf), "%08x", ntohl(sin->sin_addr.s_addr)); 2659dc3b095SChuck Lever break; 2669dc3b095SChuck Lever case AF_INET6: 267176e21eeSChuck Lever (void)rpc_ntop(sap, buf, sizeof(buf)); 268176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] = 269176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL); 2709dc3b095SChuck Lever sin6 = xs_addr_in6(xprt); 271fc0b5791SJoe Perches snprintf(buf, sizeof(buf), "%pi6", &sin6->sin6_addr); 2729dc3b095SChuck Lever break; 2739dc3b095SChuck Lever default: 2749dc3b095SChuck Lever BUG(); 2759dc3b095SChuck Lever } 276176e21eeSChuck Lever 2779dc3b095SChuck Lever xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL); 2789dc3b095SChuck Lever } 2799dc3b095SChuck Lever 2809dc3b095SChuck Lever static void xs_format_common_peer_ports(struct rpc_xprt *xprt) 2819dc3b095SChuck Lever { 2829dc3b095SChuck Lever struct sockaddr *sap = xs_addr(xprt); 2839dc3b095SChuck Lever char buf[128]; 2849dc3b095SChuck Lever 28581160e66SJoe Perches snprintf(buf, sizeof(buf), "%u", rpc_get_port(sap)); 286c877b849SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL); 287c877b849SChuck Lever 28881160e66SJoe Perches snprintf(buf, sizeof(buf), "%4hx", rpc_get_port(sap)); 289c877b849SChuck Lever xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL); 290c877b849SChuck Lever } 291c877b849SChuck Lever 2929dc3b095SChuck Lever static void xs_format_peer_addresses(struct rpc_xprt *xprt, 293b454ae90SChuck Lever const char *protocol, 294b454ae90SChuck Lever const char *netid) 295edb267a6SChuck Lever { 296b454ae90SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO] = protocol; 297b454ae90SChuck Lever xprt->address_strings[RPC_DISPLAY_NETID] = netid; 298c877b849SChuck Lever xs_format_common_peer_addresses(xprt); 2999dc3b095SChuck Lever xs_format_common_peer_ports(xprt); 300edb267a6SChuck Lever } 301edb267a6SChuck Lever 3029dc3b095SChuck Lever static void xs_update_peer_port(struct rpc_xprt *xprt) 3034b6473fbSChuck Lever { 3049dc3b095SChuck Lever kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]); 3059dc3b095SChuck Lever kfree(xprt->address_strings[RPC_DISPLAY_PORT]); 3064b6473fbSChuck Lever 3079dc3b095SChuck Lever xs_format_common_peer_ports(xprt); 308edb267a6SChuck Lever } 309edb267a6SChuck Lever 310edb267a6SChuck Lever static void xs_free_peer_addresses(struct rpc_xprt *xprt) 311edb267a6SChuck Lever { 31233e01dc7SChuck Lever unsigned int i; 31333e01dc7SChuck Lever 31433e01dc7SChuck Lever for (i = 0; i < RPC_DISPLAY_MAX; i++) 31533e01dc7SChuck Lever switch (i) { 31633e01dc7SChuck Lever case RPC_DISPLAY_PROTO: 31733e01dc7SChuck Lever case RPC_DISPLAY_NETID: 31833e01dc7SChuck Lever continue; 31933e01dc7SChuck Lever default: 32033e01dc7SChuck Lever kfree(xprt->address_strings[i]); 32133e01dc7SChuck Lever } 322edb267a6SChuck Lever } 323edb267a6SChuck Lever 324b4b5cc85SChuck Lever #define XS_SENDMSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL) 325b4b5cc85SChuck Lever 32624c5684bSTrond Myklebust static int xs_send_kvec(struct socket *sock, struct sockaddr *addr, int addrlen, struct kvec *vec, unsigned int base, int more) 327b4b5cc85SChuck Lever { 328b4b5cc85SChuck Lever struct msghdr msg = { 329b4b5cc85SChuck Lever .msg_name = addr, 330b4b5cc85SChuck Lever .msg_namelen = addrlen, 33124c5684bSTrond Myklebust .msg_flags = XS_SENDMSG_FLAGS | (more ? MSG_MORE : 0), 33224c5684bSTrond Myklebust }; 33324c5684bSTrond Myklebust struct kvec iov = { 33424c5684bSTrond Myklebust .iov_base = vec->iov_base + base, 33524c5684bSTrond Myklebust .iov_len = vec->iov_len - base, 336b4b5cc85SChuck Lever }; 337b4b5cc85SChuck Lever 33824c5684bSTrond Myklebust if (iov.iov_len != 0) 339b4b5cc85SChuck Lever return kernel_sendmsg(sock, &msg, &iov, 1, iov.iov_len); 340b4b5cc85SChuck Lever return kernel_sendmsg(sock, &msg, NULL, 0, 0); 341b4b5cc85SChuck Lever } 342b4b5cc85SChuck Lever 343f279cd00SJason Baron static int xs_send_pagedata(struct socket *sock, struct xdr_buf *xdr, unsigned int base, int more, bool zerocopy, int *sent_p) 344b4b5cc85SChuck Lever { 345a6b31d18STrond Myklebust ssize_t (*do_sendpage)(struct socket *sock, struct page *page, 346a6b31d18STrond Myklebust int offset, size_t size, int flags); 34724c5684bSTrond Myklebust struct page **ppage; 34824c5684bSTrond Myklebust unsigned int remainder; 349f279cd00SJason Baron int err; 350b4b5cc85SChuck Lever 35124c5684bSTrond Myklebust remainder = xdr->page_len - base; 35224c5684bSTrond Myklebust base += xdr->page_base; 35324c5684bSTrond Myklebust ppage = xdr->pages + (base >> PAGE_SHIFT); 35424c5684bSTrond Myklebust base &= ~PAGE_MASK; 355a6b31d18STrond Myklebust do_sendpage = sock->ops->sendpage; 356a6b31d18STrond Myklebust if (!zerocopy) 357a6b31d18STrond Myklebust do_sendpage = sock_no_sendpage; 35824c5684bSTrond Myklebust for(;;) { 35924c5684bSTrond Myklebust unsigned int len = min_t(unsigned int, PAGE_SIZE - base, remainder); 36024c5684bSTrond Myklebust int flags = XS_SENDMSG_FLAGS; 36124c5684bSTrond Myklebust 36224c5684bSTrond Myklebust remainder -= len; 36324c5684bSTrond Myklebust if (remainder != 0 || more) 36424c5684bSTrond Myklebust flags |= MSG_MORE; 365a6b31d18STrond Myklebust err = do_sendpage(sock, *ppage, base, len, flags); 36624c5684bSTrond Myklebust if (remainder == 0 || err != len) 36724c5684bSTrond Myklebust break; 368f279cd00SJason Baron *sent_p += err; 36924c5684bSTrond Myklebust ppage++; 37024c5684bSTrond Myklebust base = 0; 37124c5684bSTrond Myklebust } 372f279cd00SJason Baron if (err > 0) { 373f279cd00SJason Baron *sent_p += err; 374f279cd00SJason Baron err = 0; 375f279cd00SJason Baron } 37624c5684bSTrond Myklebust return err; 377b4b5cc85SChuck Lever } 378b4b5cc85SChuck Lever 3799903cd1cSChuck Lever /** 3809903cd1cSChuck Lever * xs_sendpages - write pages directly to a socket 3819903cd1cSChuck Lever * @sock: socket to send on 3829903cd1cSChuck Lever * @addr: UDP only -- address of destination 3839903cd1cSChuck Lever * @addrlen: UDP only -- length of destination address 3849903cd1cSChuck Lever * @xdr: buffer containing this request 3859903cd1cSChuck Lever * @base: starting position in the buffer 386a6b31d18STrond Myklebust * @zerocopy: true if it is safe to use sendpage() 387f279cd00SJason Baron * @sent_p: return the total number of bytes successfully queued for sending 3889903cd1cSChuck Lever * 389a246b010SChuck Lever */ 390f279cd00SJason Baron static int xs_sendpages(struct socket *sock, struct sockaddr *addr, int addrlen, struct xdr_buf *xdr, unsigned int base, bool zerocopy, int *sent_p) 391a246b010SChuck Lever { 39224c5684bSTrond Myklebust unsigned int remainder = xdr->len - base; 393f279cd00SJason Baron int err = 0; 394f279cd00SJason Baron int sent = 0; 395a246b010SChuck Lever 396262965f5SChuck Lever if (unlikely(!sock)) 397fba91afbSTrond Myklebust return -ENOTSOCK; 398262965f5SChuck Lever 399262965f5SChuck Lever clear_bit(SOCK_ASYNC_NOSPACE, &sock->flags); 40024c5684bSTrond Myklebust if (base != 0) { 40124c5684bSTrond Myklebust addr = NULL; 40224c5684bSTrond Myklebust addrlen = 0; 40324c5684bSTrond Myklebust } 404262965f5SChuck Lever 40524c5684bSTrond Myklebust if (base < xdr->head[0].iov_len || addr != NULL) { 40624c5684bSTrond Myklebust unsigned int len = xdr->head[0].iov_len - base; 40724c5684bSTrond Myklebust remainder -= len; 40824c5684bSTrond Myklebust err = xs_send_kvec(sock, addr, addrlen, &xdr->head[0], base, remainder != 0); 40924c5684bSTrond Myklebust if (remainder == 0 || err != len) 410a246b010SChuck Lever goto out; 411f279cd00SJason Baron *sent_p += err; 412a246b010SChuck Lever base = 0; 413a246b010SChuck Lever } else 41424c5684bSTrond Myklebust base -= xdr->head[0].iov_len; 415a246b010SChuck Lever 41624c5684bSTrond Myklebust if (base < xdr->page_len) { 41724c5684bSTrond Myklebust unsigned int len = xdr->page_len - base; 41824c5684bSTrond Myklebust remainder -= len; 419f279cd00SJason Baron err = xs_send_pagedata(sock, xdr, base, remainder != 0, zerocopy, &sent); 420f279cd00SJason Baron *sent_p += sent; 421f279cd00SJason Baron if (remainder == 0 || sent != len) 422a246b010SChuck Lever goto out; 423a246b010SChuck Lever base = 0; 42424c5684bSTrond Myklebust } else 42524c5684bSTrond Myklebust base -= xdr->page_len; 42624c5684bSTrond Myklebust 42724c5684bSTrond Myklebust if (base >= xdr->tail[0].iov_len) 428f279cd00SJason Baron return 0; 42924c5684bSTrond Myklebust err = xs_send_kvec(sock, NULL, 0, &xdr->tail[0], base, 0); 430a246b010SChuck Lever out: 431f279cd00SJason Baron if (err > 0) { 432f279cd00SJason Baron *sent_p += err; 433f279cd00SJason Baron err = 0; 434f279cd00SJason Baron } 43524c5684bSTrond Myklebust return err; 436a246b010SChuck Lever } 437a246b010SChuck Lever 438b6ddf64fSTrond Myklebust static void xs_nospace_callback(struct rpc_task *task) 439b6ddf64fSTrond Myklebust { 440b6ddf64fSTrond Myklebust struct sock_xprt *transport = container_of(task->tk_rqstp->rq_xprt, struct sock_xprt, xprt); 441b6ddf64fSTrond Myklebust 442b6ddf64fSTrond Myklebust transport->inet->sk_write_pending--; 443b6ddf64fSTrond Myklebust clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags); 444b6ddf64fSTrond Myklebust } 445b6ddf64fSTrond Myklebust 4469903cd1cSChuck Lever /** 447262965f5SChuck Lever * xs_nospace - place task on wait queue if transmit was incomplete 448262965f5SChuck Lever * @task: task to put to sleep 4499903cd1cSChuck Lever * 450a246b010SChuck Lever */ 4515e3771ceSTrond Myklebust static int xs_nospace(struct rpc_task *task) 452a246b010SChuck Lever { 453262965f5SChuck Lever struct rpc_rqst *req = task->tk_rqstp; 454262965f5SChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 455ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 45606ea0bfeSTrond Myklebust struct sock *sk = transport->inet; 45724ca9a84STrond Myklebust int ret = -EAGAIN; 458a246b010SChuck Lever 45946121cf7SChuck Lever dprintk("RPC: %5u xmit incomplete (%u left of %u)\n", 460262965f5SChuck Lever task->tk_pid, req->rq_slen - req->rq_bytes_sent, 461262965f5SChuck Lever req->rq_slen); 462a246b010SChuck Lever 463262965f5SChuck Lever /* Protect against races with write_space */ 464262965f5SChuck Lever spin_lock_bh(&xprt->transport_lock); 465a246b010SChuck Lever 466262965f5SChuck Lever /* Don't race with disconnect */ 467b6ddf64fSTrond Myklebust if (xprt_connected(xprt)) { 468b6ddf64fSTrond Myklebust if (test_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags)) { 469b6ddf64fSTrond Myklebust /* 470b6ddf64fSTrond Myklebust * Notify TCP that we're limited by the application 471b6ddf64fSTrond Myklebust * window size 472b6ddf64fSTrond Myklebust */ 473b6ddf64fSTrond Myklebust set_bit(SOCK_NOSPACE, &transport->sock->flags); 47406ea0bfeSTrond Myklebust sk->sk_write_pending++; 475b6ddf64fSTrond Myklebust /* ...and wait for more buffer space */ 476b6ddf64fSTrond Myklebust xprt_wait_for_buffer_space(task, xs_nospace_callback); 477b6ddf64fSTrond Myklebust } 478b6ddf64fSTrond Myklebust } else { 479b6ddf64fSTrond Myklebust clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags); 4805e3771ceSTrond Myklebust ret = -ENOTCONN; 481b6ddf64fSTrond Myklebust } 482a246b010SChuck Lever 483262965f5SChuck Lever spin_unlock_bh(&xprt->transport_lock); 48406ea0bfeSTrond Myklebust 48506ea0bfeSTrond Myklebust /* Race breaker in case memory is freed before above code is called */ 48606ea0bfeSTrond Myklebust sk->sk_write_space(sk); 4875e3771ceSTrond Myklebust return ret; 488a246b010SChuck Lever } 489a246b010SChuck Lever 49061677eeeSChuck Lever /* 49161677eeeSChuck Lever * Construct a stream transport record marker in @buf. 49261677eeeSChuck Lever */ 49361677eeeSChuck Lever static inline void xs_encode_stream_record_marker(struct xdr_buf *buf) 49461677eeeSChuck Lever { 49561677eeeSChuck Lever u32 reclen = buf->len - sizeof(rpc_fraghdr); 49661677eeeSChuck Lever rpc_fraghdr *base = buf->head[0].iov_base; 49761677eeeSChuck Lever *base = cpu_to_be32(RPC_LAST_STREAM_FRAGMENT | reclen); 49861677eeeSChuck Lever } 49961677eeeSChuck Lever 5009903cd1cSChuck Lever /** 501176e21eeSChuck Lever * xs_local_send_request - write an RPC request to an AF_LOCAL socket 502176e21eeSChuck Lever * @task: RPC task that manages the state of an RPC request 503176e21eeSChuck Lever * 504176e21eeSChuck Lever * Return values: 505176e21eeSChuck Lever * 0: The request has been sent 506176e21eeSChuck Lever * EAGAIN: The socket was blocked, please call again later to 507176e21eeSChuck Lever * complete the request 508176e21eeSChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again 509176e21eeSChuck Lever * other: Some other error occured, the request was not sent 510176e21eeSChuck Lever */ 511176e21eeSChuck Lever static int xs_local_send_request(struct rpc_task *task) 512176e21eeSChuck Lever { 513176e21eeSChuck Lever struct rpc_rqst *req = task->tk_rqstp; 514176e21eeSChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 515176e21eeSChuck Lever struct sock_xprt *transport = 516176e21eeSChuck Lever container_of(xprt, struct sock_xprt, xprt); 517176e21eeSChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf; 518176e21eeSChuck Lever int status; 519f279cd00SJason Baron int sent = 0; 520176e21eeSChuck Lever 521176e21eeSChuck Lever xs_encode_stream_record_marker(&req->rq_snd_buf); 522176e21eeSChuck Lever 523176e21eeSChuck Lever xs_pktdump("packet data:", 524176e21eeSChuck Lever req->rq_svec->iov_base, req->rq_svec->iov_len); 525176e21eeSChuck Lever 526f279cd00SJason Baron status = xs_sendpages(transport->sock, NULL, 0, xdr, req->rq_bytes_sent, 527f279cd00SJason Baron true, &sent); 528176e21eeSChuck Lever dprintk("RPC: %s(%u) = %d\n", 529176e21eeSChuck Lever __func__, xdr->len - req->rq_bytes_sent, status); 530743c69e7SNeilBrown 531743c69e7SNeilBrown if (status == -EAGAIN && sock_writeable(transport->inet)) 532743c69e7SNeilBrown status = -ENOBUFS; 533743c69e7SNeilBrown 534f279cd00SJason Baron if (likely(sent > 0) || status == 0) { 535f279cd00SJason Baron req->rq_bytes_sent += sent; 536f279cd00SJason Baron req->rq_xmit_bytes_sent += sent; 537176e21eeSChuck Lever if (likely(req->rq_bytes_sent >= req->rq_slen)) { 538176e21eeSChuck Lever req->rq_bytes_sent = 0; 539176e21eeSChuck Lever return 0; 540176e21eeSChuck Lever } 541176e21eeSChuck Lever status = -EAGAIN; 542176e21eeSChuck Lever } 543176e21eeSChuck Lever 544176e21eeSChuck Lever switch (status) { 5453601c4a9STrond Myklebust case -ENOBUFS: 546b5872f0cSTrond Myklebust break; 547176e21eeSChuck Lever case -EAGAIN: 548176e21eeSChuck Lever status = xs_nospace(task); 549176e21eeSChuck Lever break; 550176e21eeSChuck Lever default: 551176e21eeSChuck Lever dprintk("RPC: sendmsg returned unrecognized error %d\n", 552176e21eeSChuck Lever -status); 553176e21eeSChuck Lever case -EPIPE: 554176e21eeSChuck Lever xs_close(xprt); 555176e21eeSChuck Lever status = -ENOTCONN; 556176e21eeSChuck Lever } 557176e21eeSChuck Lever 558176e21eeSChuck Lever return status; 559176e21eeSChuck Lever } 560176e21eeSChuck Lever 561176e21eeSChuck Lever /** 562262965f5SChuck Lever * xs_udp_send_request - write an RPC request to a UDP socket 5639903cd1cSChuck Lever * @task: address of RPC task that manages the state of an RPC request 5649903cd1cSChuck Lever * 5659903cd1cSChuck Lever * Return values: 5669903cd1cSChuck Lever * 0: The request has been sent 5679903cd1cSChuck Lever * EAGAIN: The socket was blocked, please call again later to 5689903cd1cSChuck Lever * complete the request 569262965f5SChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again 57025985edcSLucas De Marchi * other: Some other error occurred, the request was not sent 5719903cd1cSChuck Lever */ 572262965f5SChuck Lever static int xs_udp_send_request(struct rpc_task *task) 573a246b010SChuck Lever { 574a246b010SChuck Lever struct rpc_rqst *req = task->tk_rqstp; 575a246b010SChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 576ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 577262965f5SChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf; 578f279cd00SJason Baron int sent = 0; 579262965f5SChuck Lever int status; 580262965f5SChuck Lever 581262965f5SChuck Lever xs_pktdump("packet data:", 582262965f5SChuck Lever req->rq_svec->iov_base, 583262965f5SChuck Lever req->rq_svec->iov_len); 584262965f5SChuck Lever 58501d37c42STrond Myklebust if (!xprt_bound(xprt)) 58601d37c42STrond Myklebust return -ENOTCONN; 587f279cd00SJason Baron status = xs_sendpages(transport->sock, xs_addr(xprt), xprt->addrlen, 588f279cd00SJason Baron xdr, req->rq_bytes_sent, true, &sent); 589262965f5SChuck Lever 590262965f5SChuck Lever dprintk("RPC: xs_udp_send_request(%u) = %d\n", 591262965f5SChuck Lever xdr->len - req->rq_bytes_sent, status); 592262965f5SChuck Lever 5933dedbb5cSJason Baron /* firewall is blocking us, don't return -EAGAIN or we end up looping */ 5943dedbb5cSJason Baron if (status == -EPERM) 5953dedbb5cSJason Baron goto process_status; 5963dedbb5cSJason Baron 597743c69e7SNeilBrown if (status == -EAGAIN && sock_writeable(transport->inet)) 598743c69e7SNeilBrown status = -ENOBUFS; 599743c69e7SNeilBrown 600f279cd00SJason Baron if (sent > 0 || status == 0) { 601f279cd00SJason Baron req->rq_xmit_bytes_sent += sent; 602f279cd00SJason Baron if (sent >= req->rq_slen) 603262965f5SChuck Lever return 0; 604262965f5SChuck Lever /* Still some bytes left; set up for a retry later. */ 605262965f5SChuck Lever status = -EAGAIN; 6062199700fSTrond Myklebust } 607262965f5SChuck Lever 6083dedbb5cSJason Baron process_status: 609262965f5SChuck Lever switch (status) { 610fba91afbSTrond Myklebust case -ENOTSOCK: 611fba91afbSTrond Myklebust status = -ENOTCONN; 612fba91afbSTrond Myklebust /* Should we call xs_close() here? */ 613fba91afbSTrond Myklebust break; 614b6ddf64fSTrond Myklebust case -EAGAIN: 6155e3771ceSTrond Myklebust status = xs_nospace(task); 616b6ddf64fSTrond Myklebust break; 617c8485e4dSTrond Myklebust default: 618c8485e4dSTrond Myklebust dprintk("RPC: sendmsg returned unrecognized error %d\n", 619c8485e4dSTrond Myklebust -status); 620262965f5SChuck Lever case -ENETUNREACH: 6213601c4a9STrond Myklebust case -ENOBUFS: 622262965f5SChuck Lever case -EPIPE: 623262965f5SChuck Lever case -ECONNREFUSED: 6243dedbb5cSJason Baron case -EPERM: 625262965f5SChuck Lever /* When the server has died, an ICMP port unreachable message 626262965f5SChuck Lever * prompts ECONNREFUSED. */ 627b6ddf64fSTrond Myklebust clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags); 628262965f5SChuck Lever } 6295fe46e9dSBian Naimeng 630262965f5SChuck Lever return status; 631262965f5SChuck Lever } 632262965f5SChuck Lever 633e06799f9STrond Myklebust /** 634262965f5SChuck Lever * xs_tcp_send_request - write an RPC request to a TCP socket 635262965f5SChuck Lever * @task: address of RPC task that manages the state of an RPC request 636262965f5SChuck Lever * 637262965f5SChuck Lever * Return values: 638262965f5SChuck Lever * 0: The request has been sent 639262965f5SChuck Lever * EAGAIN: The socket was blocked, please call again later to 640262965f5SChuck Lever * complete the request 641262965f5SChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again 64225985edcSLucas De Marchi * other: Some other error occurred, the request was not sent 643262965f5SChuck Lever * 644262965f5SChuck Lever * XXX: In the case of soft timeouts, should we eventually give up 645262965f5SChuck Lever * if sendmsg is not able to make progress? 646262965f5SChuck Lever */ 647262965f5SChuck Lever static int xs_tcp_send_request(struct rpc_task *task) 648262965f5SChuck Lever { 649262965f5SChuck Lever struct rpc_rqst *req = task->tk_rqstp; 650262965f5SChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 651ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 652262965f5SChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf; 653a6b31d18STrond Myklebust bool zerocopy = true; 654b595bb15SChuck Lever int status; 655f279cd00SJason Baron int sent; 656a246b010SChuck Lever 65761677eeeSChuck Lever xs_encode_stream_record_marker(&req->rq_snd_buf); 658262965f5SChuck Lever 659262965f5SChuck Lever xs_pktdump("packet data:", 660262965f5SChuck Lever req->rq_svec->iov_base, 661262965f5SChuck Lever req->rq_svec->iov_len); 662a6b31d18STrond Myklebust /* Don't use zero copy if this is a resend. If the RPC call 663a6b31d18STrond Myklebust * completes while the socket holds a reference to the pages, 664a6b31d18STrond Myklebust * then we may end up resending corrupted data. 665a6b31d18STrond Myklebust */ 666a6b31d18STrond Myklebust if (task->tk_flags & RPC_TASK_SENT) 667a6b31d18STrond Myklebust zerocopy = false; 668a246b010SChuck Lever 669a246b010SChuck Lever /* Continue transmitting the packet/record. We must be careful 670a246b010SChuck Lever * to cope with writespace callbacks arriving _after_ we have 671262965f5SChuck Lever * called sendmsg(). */ 672a246b010SChuck Lever while (1) { 673f279cd00SJason Baron sent = 0; 674f279cd00SJason Baron status = xs_sendpages(transport->sock, NULL, 0, xdr, 675f279cd00SJason Baron req->rq_bytes_sent, zerocopy, &sent); 676a246b010SChuck Lever 677262965f5SChuck Lever dprintk("RPC: xs_tcp_send_request(%u) = %d\n", 678262965f5SChuck Lever xdr->len - req->rq_bytes_sent, status); 679262965f5SChuck Lever 680a246b010SChuck Lever /* If we've sent the entire packet, immediately 681a246b010SChuck Lever * reset the count of bytes sent. */ 682f279cd00SJason Baron req->rq_bytes_sent += sent; 683f279cd00SJason Baron req->rq_xmit_bytes_sent += sent; 684262965f5SChuck Lever if (likely(req->rq_bytes_sent >= req->rq_slen)) { 685a246b010SChuck Lever req->rq_bytes_sent = 0; 686a246b010SChuck Lever return 0; 687a246b010SChuck Lever } 688262965f5SChuck Lever 689f580dd04STrond Myklebust if (status < 0) 690f580dd04STrond Myklebust break; 691f580dd04STrond Myklebust if (sent == 0) { 692a246b010SChuck Lever status = -EAGAIN; 693a246b010SChuck Lever break; 694a246b010SChuck Lever } 695f580dd04STrond Myklebust } 696743c69e7SNeilBrown if (status == -EAGAIN && sk_stream_is_writeable(transport->inet)) 697743c69e7SNeilBrown status = -ENOBUFS; 698a246b010SChuck Lever 699262965f5SChuck Lever switch (status) { 700fba91afbSTrond Myklebust case -ENOTSOCK: 701fba91afbSTrond Myklebust status = -ENOTCONN; 702fba91afbSTrond Myklebust /* Should we call xs_close() here? */ 703fba91afbSTrond Myklebust break; 704262965f5SChuck Lever case -EAGAIN: 7055e3771ceSTrond Myklebust status = xs_nospace(task); 706262965f5SChuck Lever break; 707262965f5SChuck Lever default: 708262965f5SChuck Lever dprintk("RPC: sendmsg returned unrecognized error %d\n", 709262965f5SChuck Lever -status); 710a246b010SChuck Lever case -ECONNRESET: 711a246b010SChuck Lever case -ECONNREFUSED: 712a246b010SChuck Lever case -ENOTCONN: 7133913c78cSTrond Myklebust case -EADDRINUSE: 714b5872f0cSTrond Myklebust case -ENOBUFS: 715b9d2bb2eSTrond Myklebust case -EPIPE: 716a246b010SChuck Lever clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags); 717a246b010SChuck Lever } 7185fe46e9dSBian Naimeng 719a246b010SChuck Lever return status; 720a246b010SChuck Lever } 721a246b010SChuck Lever 7229903cd1cSChuck Lever /** 723e0ab53deSTrond Myklebust * xs_tcp_release_xprt - clean up after a tcp transmission 724e0ab53deSTrond Myklebust * @xprt: transport 725e0ab53deSTrond Myklebust * @task: rpc task 726e0ab53deSTrond Myklebust * 727e0ab53deSTrond Myklebust * This cleans up if an error causes us to abort the transmission of a request. 728e0ab53deSTrond Myklebust * In this case, the socket may need to be reset in order to avoid confusing 729e0ab53deSTrond Myklebust * the server. 730e0ab53deSTrond Myklebust */ 731e0ab53deSTrond Myklebust static void xs_tcp_release_xprt(struct rpc_xprt *xprt, struct rpc_task *task) 732e0ab53deSTrond Myklebust { 733e0ab53deSTrond Myklebust struct rpc_rqst *req; 734e0ab53deSTrond Myklebust 735e0ab53deSTrond Myklebust if (task != xprt->snd_task) 736e0ab53deSTrond Myklebust return; 737e0ab53deSTrond Myklebust if (task == NULL) 738e0ab53deSTrond Myklebust goto out_release; 739e0ab53deSTrond Myklebust req = task->tk_rqstp; 74043cedbf0STrond Myklebust if (req == NULL) 74143cedbf0STrond Myklebust goto out_release; 742e0ab53deSTrond Myklebust if (req->rq_bytes_sent == 0) 743e0ab53deSTrond Myklebust goto out_release; 744e0ab53deSTrond Myklebust if (req->rq_bytes_sent == req->rq_snd_buf.len) 745e0ab53deSTrond Myklebust goto out_release; 746a4f0835cSTrond Myklebust set_bit(XPRT_CLOSE_WAIT, &xprt->state); 747e0ab53deSTrond Myklebust out_release: 748e0ab53deSTrond Myklebust xprt_release_xprt(xprt, task); 749e0ab53deSTrond Myklebust } 750e0ab53deSTrond Myklebust 7512a9e1cfaSTrond Myklebust static void xs_save_old_callbacks(struct sock_xprt *transport, struct sock *sk) 7522a9e1cfaSTrond Myklebust { 7532a9e1cfaSTrond Myklebust transport->old_data_ready = sk->sk_data_ready; 7542a9e1cfaSTrond Myklebust transport->old_state_change = sk->sk_state_change; 7552a9e1cfaSTrond Myklebust transport->old_write_space = sk->sk_write_space; 7562118071dSTrond Myklebust transport->old_error_report = sk->sk_error_report; 7572a9e1cfaSTrond Myklebust } 7582a9e1cfaSTrond Myklebust 7592a9e1cfaSTrond Myklebust static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *sk) 7602a9e1cfaSTrond Myklebust { 7612a9e1cfaSTrond Myklebust sk->sk_data_ready = transport->old_data_ready; 7622a9e1cfaSTrond Myklebust sk->sk_state_change = transport->old_state_change; 7632a9e1cfaSTrond Myklebust sk->sk_write_space = transport->old_write_space; 7642118071dSTrond Myklebust sk->sk_error_report = transport->old_error_report; 7652118071dSTrond Myklebust } 7662118071dSTrond Myklebust 767b70ae915STrond Myklebust static void xs_sock_reset_connection_flags(struct rpc_xprt *xprt) 768b70ae915STrond Myklebust { 769b70ae915STrond Myklebust smp_mb__before_atomic(); 770b70ae915STrond Myklebust clear_bit(XPRT_CLOSE_WAIT, &xprt->state); 771b70ae915STrond Myklebust clear_bit(XPRT_CLOSING, &xprt->state); 772b70ae915STrond Myklebust smp_mb__after_atomic(); 773b70ae915STrond Myklebust } 774b70ae915STrond Myklebust 775b70ae915STrond Myklebust static void xs_sock_mark_closed(struct rpc_xprt *xprt) 776b70ae915STrond Myklebust { 777b70ae915STrond Myklebust xs_sock_reset_connection_flags(xprt); 778b70ae915STrond Myklebust /* Mark transport as closed and wake up all pending tasks */ 779b70ae915STrond Myklebust xprt_disconnect_done(xprt); 780b70ae915STrond Myklebust } 781b70ae915STrond Myklebust 7822118071dSTrond Myklebust /** 7832118071dSTrond Myklebust * xs_error_report - callback to handle TCP socket state errors 7842118071dSTrond Myklebust * @sk: socket 7852118071dSTrond Myklebust * 7862118071dSTrond Myklebust * Note: we don't call sock_error() since there may be a rpc_task 7872118071dSTrond Myklebust * using the socket, and so we don't want to clear sk->sk_err. 7882118071dSTrond Myklebust */ 7892118071dSTrond Myklebust static void xs_error_report(struct sock *sk) 7902118071dSTrond Myklebust { 7912118071dSTrond Myklebust struct rpc_xprt *xprt; 7922118071dSTrond Myklebust int err; 7932118071dSTrond Myklebust 7942118071dSTrond Myklebust read_lock_bh(&sk->sk_callback_lock); 7952118071dSTrond Myklebust if (!(xprt = xprt_from_sock(sk))) 7962118071dSTrond Myklebust goto out; 7972118071dSTrond Myklebust 7982118071dSTrond Myklebust err = -sk->sk_err; 7992118071dSTrond Myklebust if (err == 0) 8002118071dSTrond Myklebust goto out; 801b70ae915STrond Myklebust /* Is this a reset event? */ 802b70ae915STrond Myklebust if (sk->sk_state == TCP_CLOSE) 803b70ae915STrond Myklebust xs_sock_mark_closed(xprt); 8042118071dSTrond Myklebust dprintk("RPC: xs_error_report client %p, error=%d...\n", 8052118071dSTrond Myklebust xprt, -err); 806e8353c76STrond Myklebust trace_rpc_socket_error(xprt, sk->sk_socket, err); 8072118071dSTrond Myklebust xprt_wake_pending_tasks(xprt, err); 8082118071dSTrond Myklebust out: 8092118071dSTrond Myklebust read_unlock_bh(&sk->sk_callback_lock); 8102a9e1cfaSTrond Myklebust } 8112a9e1cfaSTrond Myklebust 812fe315e76SChuck Lever static void xs_reset_transport(struct sock_xprt *transport) 813a246b010SChuck Lever { 814ee0ac0c2SChuck Lever struct socket *sock = transport->sock; 815ee0ac0c2SChuck Lever struct sock *sk = transport->inet; 8166cc7e908STrond Myklebust struct rpc_xprt *xprt = &transport->xprt; 817a246b010SChuck Lever 818fe315e76SChuck Lever if (sk == NULL) 819fe315e76SChuck Lever return; 8209903cd1cSChuck Lever 821264d1df3SJeff Layton if (atomic_read(&transport->xprt.swapper)) 822264d1df3SJeff Layton sk_clear_memalloc(sk); 823264d1df3SJeff Layton 82409939204STrond Myklebust kernel_sock_shutdown(sock, SHUT_RDWR); 82509939204STrond Myklebust 826*edc1b01cSTrond Myklebust mutex_lock(&transport->recv_mutex); 827a246b010SChuck Lever write_lock_bh(&sk->sk_callback_lock); 828ee0ac0c2SChuck Lever transport->inet = NULL; 829ee0ac0c2SChuck Lever transport->sock = NULL; 830a246b010SChuck Lever 831a246b010SChuck Lever sk->sk_user_data = NULL; 8322a9e1cfaSTrond Myklebust 8332a9e1cfaSTrond Myklebust xs_restore_old_callbacks(transport, sk); 8340c78789eSTrond Myklebust xprt_clear_connected(xprt); 835a246b010SChuck Lever write_unlock_bh(&sk->sk_callback_lock); 8366cc7e908STrond Myklebust xs_sock_reset_connection_flags(xprt); 837*edc1b01cSTrond Myklebust mutex_unlock(&transport->recv_mutex); 838a246b010SChuck Lever 8396cc7e908STrond Myklebust trace_rpc_socket_close(xprt, sock); 840a246b010SChuck Lever sock_release(sock); 841fe315e76SChuck Lever } 842fe315e76SChuck Lever 843fe315e76SChuck Lever /** 844fe315e76SChuck Lever * xs_close - close a socket 845fe315e76SChuck Lever * @xprt: transport 846fe315e76SChuck Lever * 847fe315e76SChuck Lever * This is used when all requests are complete; ie, no DRC state remains 848fe315e76SChuck Lever * on the server we want to save. 849f75e6745STrond Myklebust * 850f75e6745STrond Myklebust * The caller _must_ be holding XPRT_LOCKED in order to avoid issues with 851f75e6745STrond Myklebust * xs_reset_transport() zeroing the socket from underneath a writer. 852fe315e76SChuck Lever */ 853fe315e76SChuck Lever static void xs_close(struct rpc_xprt *xprt) 854fe315e76SChuck Lever { 855fe315e76SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 856fe315e76SChuck Lever 857fe315e76SChuck Lever dprintk("RPC: xs_close xprt %p\n", xprt); 858fe315e76SChuck Lever 859fe315e76SChuck Lever xs_reset_transport(transport); 86061d0a8e6SNeil Brown xprt->reestablish_timeout = 0; 861fe315e76SChuck Lever 86262da3b24STrond Myklebust xprt_disconnect_done(xprt); 863a246b010SChuck Lever } 864a246b010SChuck Lever 8654a068258SChuck Lever static void xs_inject_disconnect(struct rpc_xprt *xprt) 8664a068258SChuck Lever { 8674a068258SChuck Lever dprintk("RPC: injecting transport disconnect on xprt=%p\n", 8684a068258SChuck Lever xprt); 8694a068258SChuck Lever xprt_disconnect_done(xprt); 8704a068258SChuck Lever } 8714a068258SChuck Lever 872315f3812SKinglong Mee static void xs_xprt_free(struct rpc_xprt *xprt) 873315f3812SKinglong Mee { 874315f3812SKinglong Mee xs_free_peer_addresses(xprt); 875315f3812SKinglong Mee xprt_free(xprt); 876315f3812SKinglong Mee } 877315f3812SKinglong Mee 8789903cd1cSChuck Lever /** 8799903cd1cSChuck Lever * xs_destroy - prepare to shutdown a transport 8809903cd1cSChuck Lever * @xprt: doomed transport 8819903cd1cSChuck Lever * 8829903cd1cSChuck Lever */ 8839903cd1cSChuck Lever static void xs_destroy(struct rpc_xprt *xprt) 884a246b010SChuck Lever { 88503c78827STrond Myklebust struct sock_xprt *transport = container_of(xprt, 88603c78827STrond Myklebust struct sock_xprt, xprt); 8879903cd1cSChuck Lever dprintk("RPC: xs_destroy xprt %p\n", xprt); 8889903cd1cSChuck Lever 88903c78827STrond Myklebust cancel_delayed_work_sync(&transport->connect_worker); 890a1311d87STrond Myklebust xs_close(xprt); 891*edc1b01cSTrond Myklebust cancel_work_sync(&transport->recv_worker); 892315f3812SKinglong Mee xs_xprt_free(xprt); 893a1311d87STrond Myklebust module_put(THIS_MODULE); 894a246b010SChuck Lever } 895a246b010SChuck Lever 896176e21eeSChuck Lever static int xs_local_copy_to_xdr(struct xdr_buf *xdr, struct sk_buff *skb) 897176e21eeSChuck Lever { 898176e21eeSChuck Lever struct xdr_skb_reader desc = { 899176e21eeSChuck Lever .skb = skb, 900176e21eeSChuck Lever .offset = sizeof(rpc_fraghdr), 901176e21eeSChuck Lever .count = skb->len - sizeof(rpc_fraghdr), 902176e21eeSChuck Lever }; 903176e21eeSChuck Lever 904176e21eeSChuck Lever if (xdr_partial_copy_from_skb(xdr, 0, &desc, xdr_skb_read_bits) < 0) 905176e21eeSChuck Lever return -1; 906176e21eeSChuck Lever if (desc.count) 907176e21eeSChuck Lever return -1; 908176e21eeSChuck Lever return 0; 909176e21eeSChuck Lever } 910176e21eeSChuck Lever 911176e21eeSChuck Lever /** 912176e21eeSChuck Lever * xs_local_data_ready - "data ready" callback for AF_LOCAL sockets 913176e21eeSChuck Lever * @sk: socket with data to read 914176e21eeSChuck Lever * 915176e21eeSChuck Lever * Currently this assumes we can read the whole reply in a single gulp. 916176e21eeSChuck Lever */ 917676d2369SDavid S. Miller static void xs_local_data_ready(struct sock *sk) 918176e21eeSChuck Lever { 919176e21eeSChuck Lever struct rpc_task *task; 920176e21eeSChuck Lever struct rpc_xprt *xprt; 921176e21eeSChuck Lever struct rpc_rqst *rovr; 922176e21eeSChuck Lever struct sk_buff *skb; 923176e21eeSChuck Lever int err, repsize, copied; 924176e21eeSChuck Lever u32 _xid; 925176e21eeSChuck Lever __be32 *xp; 926176e21eeSChuck Lever 927176e21eeSChuck Lever read_lock_bh(&sk->sk_callback_lock); 928176e21eeSChuck Lever dprintk("RPC: %s...\n", __func__); 929176e21eeSChuck Lever xprt = xprt_from_sock(sk); 930176e21eeSChuck Lever if (xprt == NULL) 931176e21eeSChuck Lever goto out; 932176e21eeSChuck Lever 933176e21eeSChuck Lever skb = skb_recv_datagram(sk, 0, 1, &err); 934176e21eeSChuck Lever if (skb == NULL) 935176e21eeSChuck Lever goto out; 936176e21eeSChuck Lever 937176e21eeSChuck Lever repsize = skb->len - sizeof(rpc_fraghdr); 938176e21eeSChuck Lever if (repsize < 4) { 939176e21eeSChuck Lever dprintk("RPC: impossible RPC reply size %d\n", repsize); 940176e21eeSChuck Lever goto dropit; 941176e21eeSChuck Lever } 942176e21eeSChuck Lever 943176e21eeSChuck Lever /* Copy the XID from the skb... */ 944176e21eeSChuck Lever xp = skb_header_pointer(skb, sizeof(rpc_fraghdr), sizeof(_xid), &_xid); 945176e21eeSChuck Lever if (xp == NULL) 946176e21eeSChuck Lever goto dropit; 947176e21eeSChuck Lever 948176e21eeSChuck Lever /* Look up and lock the request corresponding to the given XID */ 949176e21eeSChuck Lever spin_lock(&xprt->transport_lock); 950176e21eeSChuck Lever rovr = xprt_lookup_rqst(xprt, *xp); 951176e21eeSChuck Lever if (!rovr) 952176e21eeSChuck Lever goto out_unlock; 953176e21eeSChuck Lever task = rovr->rq_task; 954176e21eeSChuck Lever 955176e21eeSChuck Lever copied = rovr->rq_private_buf.buflen; 956176e21eeSChuck Lever if (copied > repsize) 957176e21eeSChuck Lever copied = repsize; 958176e21eeSChuck Lever 959176e21eeSChuck Lever if (xs_local_copy_to_xdr(&rovr->rq_private_buf, skb)) { 960176e21eeSChuck Lever dprintk("RPC: sk_buff copy failed\n"); 961176e21eeSChuck Lever goto out_unlock; 962176e21eeSChuck Lever } 963176e21eeSChuck Lever 964176e21eeSChuck Lever xprt_complete_rqst(task, copied); 965176e21eeSChuck Lever 966176e21eeSChuck Lever out_unlock: 967176e21eeSChuck Lever spin_unlock(&xprt->transport_lock); 968176e21eeSChuck Lever dropit: 969176e21eeSChuck Lever skb_free_datagram(sk, skb); 970176e21eeSChuck Lever out: 971176e21eeSChuck Lever read_unlock_bh(&sk->sk_callback_lock); 972176e21eeSChuck Lever } 973176e21eeSChuck Lever 9749903cd1cSChuck Lever /** 9759903cd1cSChuck Lever * xs_udp_data_ready - "data ready" callback for UDP sockets 9769903cd1cSChuck Lever * @sk: socket with data to read 9779903cd1cSChuck Lever * 978a246b010SChuck Lever */ 979676d2369SDavid S. Miller static void xs_udp_data_ready(struct sock *sk) 980a246b010SChuck Lever { 981a246b010SChuck Lever struct rpc_task *task; 982a246b010SChuck Lever struct rpc_xprt *xprt; 983a246b010SChuck Lever struct rpc_rqst *rovr; 984a246b010SChuck Lever struct sk_buff *skb; 985a246b010SChuck Lever int err, repsize, copied; 986d8ed029dSAlexey Dobriyan u32 _xid; 987d8ed029dSAlexey Dobriyan __be32 *xp; 988a246b010SChuck Lever 989f064af1eSEric Dumazet read_lock_bh(&sk->sk_callback_lock); 9909903cd1cSChuck Lever dprintk("RPC: xs_udp_data_ready...\n"); 9919903cd1cSChuck Lever if (!(xprt = xprt_from_sock(sk))) 992a246b010SChuck Lever goto out; 993a246b010SChuck Lever 994a246b010SChuck Lever if ((skb = skb_recv_datagram(sk, 0, 1, &err)) == NULL) 995a246b010SChuck Lever goto out; 996a246b010SChuck Lever 997a246b010SChuck Lever repsize = skb->len - sizeof(struct udphdr); 998a246b010SChuck Lever if (repsize < 4) { 9999903cd1cSChuck Lever dprintk("RPC: impossible RPC reply size %d!\n", repsize); 1000a246b010SChuck Lever goto dropit; 1001a246b010SChuck Lever } 1002a246b010SChuck Lever 1003a246b010SChuck Lever /* Copy the XID from the skb... */ 1004a246b010SChuck Lever xp = skb_header_pointer(skb, sizeof(struct udphdr), 1005a246b010SChuck Lever sizeof(_xid), &_xid); 1006a246b010SChuck Lever if (xp == NULL) 1007a246b010SChuck Lever goto dropit; 1008a246b010SChuck Lever 1009a246b010SChuck Lever /* Look up and lock the request corresponding to the given XID */ 10104a0f8c04SChuck Lever spin_lock(&xprt->transport_lock); 1011a246b010SChuck Lever rovr = xprt_lookup_rqst(xprt, *xp); 1012a246b010SChuck Lever if (!rovr) 1013a246b010SChuck Lever goto out_unlock; 1014a246b010SChuck Lever task = rovr->rq_task; 1015a246b010SChuck Lever 1016a246b010SChuck Lever if ((copied = rovr->rq_private_buf.buflen) > repsize) 1017a246b010SChuck Lever copied = repsize; 1018a246b010SChuck Lever 1019a246b010SChuck Lever /* Suck it into the iovec, verify checksum if not done by hw. */ 10201781f7f5SHerbert Xu if (csum_partial_copy_to_xdr(&rovr->rq_private_buf, skb)) { 10211781f7f5SHerbert Xu UDPX_INC_STATS_BH(sk, UDP_MIB_INERRORS); 1022a246b010SChuck Lever goto out_unlock; 10231781f7f5SHerbert Xu } 10241781f7f5SHerbert Xu 10251781f7f5SHerbert Xu UDPX_INC_STATS_BH(sk, UDP_MIB_INDATAGRAMS); 1026a246b010SChuck Lever 10276a24dfb6STrond Myklebust xprt_adjust_cwnd(xprt, task, copied); 10281570c1e4SChuck Lever xprt_complete_rqst(task, copied); 1029a246b010SChuck Lever 1030a246b010SChuck Lever out_unlock: 10314a0f8c04SChuck Lever spin_unlock(&xprt->transport_lock); 1032a246b010SChuck Lever dropit: 1033a246b010SChuck Lever skb_free_datagram(sk, skb); 1034a246b010SChuck Lever out: 1035f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1036a246b010SChuck Lever } 1037a246b010SChuck Lever 1038a519fc7aSTrond Myklebust /* 1039a519fc7aSTrond Myklebust * Helper function to force a TCP close if the server is sending 1040a519fc7aSTrond Myklebust * junk and/or it has put us in CLOSE_WAIT 1041a519fc7aSTrond Myklebust */ 1042a519fc7aSTrond Myklebust static void xs_tcp_force_close(struct rpc_xprt *xprt) 1043a519fc7aSTrond Myklebust { 1044a519fc7aSTrond Myklebust xprt_force_disconnect(xprt); 1045a519fc7aSTrond Myklebust } 1046a519fc7aSTrond Myklebust 1047dd456471SChuck Lever static inline void xs_tcp_read_fraghdr(struct rpc_xprt *xprt, struct xdr_skb_reader *desc) 1048a246b010SChuck Lever { 104951971139SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1050a246b010SChuck Lever size_t len, used; 1051a246b010SChuck Lever char *p; 1052a246b010SChuck Lever 105351971139SChuck Lever p = ((char *) &transport->tcp_fraghdr) + transport->tcp_offset; 105451971139SChuck Lever len = sizeof(transport->tcp_fraghdr) - transport->tcp_offset; 10559d292316SChuck Lever used = xdr_skb_read_bits(desc, p, len); 105651971139SChuck Lever transport->tcp_offset += used; 1057a246b010SChuck Lever if (used != len) 1058a246b010SChuck Lever return; 1059808012fbSChuck Lever 106051971139SChuck Lever transport->tcp_reclen = ntohl(transport->tcp_fraghdr); 106151971139SChuck Lever if (transport->tcp_reclen & RPC_LAST_STREAM_FRAGMENT) 1062e136d092SChuck Lever transport->tcp_flags |= TCP_RCV_LAST_FRAG; 1063a246b010SChuck Lever else 1064e136d092SChuck Lever transport->tcp_flags &= ~TCP_RCV_LAST_FRAG; 106551971139SChuck Lever transport->tcp_reclen &= RPC_FRAGMENT_SIZE_MASK; 1066808012fbSChuck Lever 1067e136d092SChuck Lever transport->tcp_flags &= ~TCP_RCV_COPY_FRAGHDR; 106851971139SChuck Lever transport->tcp_offset = 0; 1069808012fbSChuck Lever 1070a246b010SChuck Lever /* Sanity check of the record length */ 107118dca02aSRicardo Labiaga if (unlikely(transport->tcp_reclen < 8)) { 10729903cd1cSChuck Lever dprintk("RPC: invalid TCP record fragment length\n"); 1073a519fc7aSTrond Myklebust xs_tcp_force_close(xprt); 10749903cd1cSChuck Lever return; 1075a246b010SChuck Lever } 1076a246b010SChuck Lever dprintk("RPC: reading TCP record fragment of length %d\n", 107751971139SChuck Lever transport->tcp_reclen); 1078a246b010SChuck Lever } 1079a246b010SChuck Lever 108051971139SChuck Lever static void xs_tcp_check_fraghdr(struct sock_xprt *transport) 1081a246b010SChuck Lever { 108251971139SChuck Lever if (transport->tcp_offset == transport->tcp_reclen) { 1083e136d092SChuck Lever transport->tcp_flags |= TCP_RCV_COPY_FRAGHDR; 108451971139SChuck Lever transport->tcp_offset = 0; 1085e136d092SChuck Lever if (transport->tcp_flags & TCP_RCV_LAST_FRAG) { 1086e136d092SChuck Lever transport->tcp_flags &= ~TCP_RCV_COPY_DATA; 1087e136d092SChuck Lever transport->tcp_flags |= TCP_RCV_COPY_XID; 108851971139SChuck Lever transport->tcp_copied = 0; 1089a246b010SChuck Lever } 1090a246b010SChuck Lever } 1091a246b010SChuck Lever } 1092a246b010SChuck Lever 1093dd456471SChuck Lever static inline void xs_tcp_read_xid(struct sock_xprt *transport, struct xdr_skb_reader *desc) 1094a246b010SChuck Lever { 1095a246b010SChuck Lever size_t len, used; 1096a246b010SChuck Lever char *p; 1097a246b010SChuck Lever 109851971139SChuck Lever len = sizeof(transport->tcp_xid) - transport->tcp_offset; 1099a246b010SChuck Lever dprintk("RPC: reading XID (%Zu bytes)\n", len); 110051971139SChuck Lever p = ((char *) &transport->tcp_xid) + transport->tcp_offset; 11019d292316SChuck Lever used = xdr_skb_read_bits(desc, p, len); 110251971139SChuck Lever transport->tcp_offset += used; 1103a246b010SChuck Lever if (used != len) 1104a246b010SChuck Lever return; 1105e136d092SChuck Lever transport->tcp_flags &= ~TCP_RCV_COPY_XID; 1106f4a2e418SRicardo Labiaga transport->tcp_flags |= TCP_RCV_READ_CALLDIR; 110751971139SChuck Lever transport->tcp_copied = 4; 110818dca02aSRicardo Labiaga dprintk("RPC: reading %s XID %08x\n", 110918dca02aSRicardo Labiaga (transport->tcp_flags & TCP_RPC_REPLY) ? "reply for" 111018dca02aSRicardo Labiaga : "request with", 111151971139SChuck Lever ntohl(transport->tcp_xid)); 111251971139SChuck Lever xs_tcp_check_fraghdr(transport); 1113a246b010SChuck Lever } 1114a246b010SChuck Lever 111518dca02aSRicardo Labiaga static inline void xs_tcp_read_calldir(struct sock_xprt *transport, 111618dca02aSRicardo Labiaga struct xdr_skb_reader *desc) 1117a246b010SChuck Lever { 111818dca02aSRicardo Labiaga size_t len, used; 111918dca02aSRicardo Labiaga u32 offset; 1120b76ce561STrond Myklebust char *p; 112118dca02aSRicardo Labiaga 112218dca02aSRicardo Labiaga /* 112318dca02aSRicardo Labiaga * We want transport->tcp_offset to be 8 at the end of this routine 112418dca02aSRicardo Labiaga * (4 bytes for the xid and 4 bytes for the call/reply flag). 112518dca02aSRicardo Labiaga * When this function is called for the first time, 112618dca02aSRicardo Labiaga * transport->tcp_offset is 4 (after having already read the xid). 112718dca02aSRicardo Labiaga */ 112818dca02aSRicardo Labiaga offset = transport->tcp_offset - sizeof(transport->tcp_xid); 1129b76ce561STrond Myklebust len = sizeof(transport->tcp_calldir) - offset; 113018dca02aSRicardo Labiaga dprintk("RPC: reading CALL/REPLY flag (%Zu bytes)\n", len); 1131b76ce561STrond Myklebust p = ((char *) &transport->tcp_calldir) + offset; 1132b76ce561STrond Myklebust used = xdr_skb_read_bits(desc, p, len); 113318dca02aSRicardo Labiaga transport->tcp_offset += used; 113418dca02aSRicardo Labiaga if (used != len) 113518dca02aSRicardo Labiaga return; 1136f4a2e418SRicardo Labiaga transport->tcp_flags &= ~TCP_RCV_READ_CALLDIR; 1137f4a2e418SRicardo Labiaga /* 1138f4a2e418SRicardo Labiaga * We don't yet have the XDR buffer, so we will write the calldir 1139f4a2e418SRicardo Labiaga * out after we get the buffer from the 'struct rpc_rqst' 1140f4a2e418SRicardo Labiaga */ 1141b76ce561STrond Myklebust switch (ntohl(transport->tcp_calldir)) { 1142b76ce561STrond Myklebust case RPC_REPLY: 1143b76ce561STrond Myklebust transport->tcp_flags |= TCP_RCV_COPY_CALLDIR; 1144b76ce561STrond Myklebust transport->tcp_flags |= TCP_RCV_COPY_DATA; 114518dca02aSRicardo Labiaga transport->tcp_flags |= TCP_RPC_REPLY; 1146b76ce561STrond Myklebust break; 1147b76ce561STrond Myklebust case RPC_CALL: 1148b76ce561STrond Myklebust transport->tcp_flags |= TCP_RCV_COPY_CALLDIR; 1149b76ce561STrond Myklebust transport->tcp_flags |= TCP_RCV_COPY_DATA; 115018dca02aSRicardo Labiaga transport->tcp_flags &= ~TCP_RPC_REPLY; 1151b76ce561STrond Myklebust break; 1152b76ce561STrond Myklebust default: 1153b76ce561STrond Myklebust dprintk("RPC: invalid request message type\n"); 1154a519fc7aSTrond Myklebust xs_tcp_force_close(&transport->xprt); 1155b76ce561STrond Myklebust } 115618dca02aSRicardo Labiaga xs_tcp_check_fraghdr(transport); 115718dca02aSRicardo Labiaga } 115818dca02aSRicardo Labiaga 115944b98efdSRicardo Labiaga static inline void xs_tcp_read_common(struct rpc_xprt *xprt, 116044b98efdSRicardo Labiaga struct xdr_skb_reader *desc, 116144b98efdSRicardo Labiaga struct rpc_rqst *req) 1162a246b010SChuck Lever { 116344b98efdSRicardo Labiaga struct sock_xprt *transport = 116444b98efdSRicardo Labiaga container_of(xprt, struct sock_xprt, xprt); 1165a246b010SChuck Lever struct xdr_buf *rcvbuf; 1166a246b010SChuck Lever size_t len; 1167a246b010SChuck Lever ssize_t r; 1168a246b010SChuck Lever 1169a246b010SChuck Lever rcvbuf = &req->rq_private_buf; 1170f4a2e418SRicardo Labiaga 1171f4a2e418SRicardo Labiaga if (transport->tcp_flags & TCP_RCV_COPY_CALLDIR) { 1172f4a2e418SRicardo Labiaga /* 1173f4a2e418SRicardo Labiaga * Save the RPC direction in the XDR buffer 1174f4a2e418SRicardo Labiaga */ 1175f4a2e418SRicardo Labiaga memcpy(rcvbuf->head[0].iov_base + transport->tcp_copied, 1176b76ce561STrond Myklebust &transport->tcp_calldir, 1177b76ce561STrond Myklebust sizeof(transport->tcp_calldir)); 1178b76ce561STrond Myklebust transport->tcp_copied += sizeof(transport->tcp_calldir); 1179f4a2e418SRicardo Labiaga transport->tcp_flags &= ~TCP_RCV_COPY_CALLDIR; 1180a246b010SChuck Lever } 1181a246b010SChuck Lever 1182a246b010SChuck Lever len = desc->count; 118351971139SChuck Lever if (len > transport->tcp_reclen - transport->tcp_offset) { 1184dd456471SChuck Lever struct xdr_skb_reader my_desc; 1185a246b010SChuck Lever 118651971139SChuck Lever len = transport->tcp_reclen - transport->tcp_offset; 1187a246b010SChuck Lever memcpy(&my_desc, desc, sizeof(my_desc)); 1188a246b010SChuck Lever my_desc.count = len; 118951971139SChuck Lever r = xdr_partial_copy_from_skb(rcvbuf, transport->tcp_copied, 11909d292316SChuck Lever &my_desc, xdr_skb_read_bits); 1191a246b010SChuck Lever desc->count -= r; 1192a246b010SChuck Lever desc->offset += r; 1193a246b010SChuck Lever } else 119451971139SChuck Lever r = xdr_partial_copy_from_skb(rcvbuf, transport->tcp_copied, 11959d292316SChuck Lever desc, xdr_skb_read_bits); 1196a246b010SChuck Lever 1197a246b010SChuck Lever if (r > 0) { 119851971139SChuck Lever transport->tcp_copied += r; 119951971139SChuck Lever transport->tcp_offset += r; 1200a246b010SChuck Lever } 1201a246b010SChuck Lever if (r != len) { 1202a246b010SChuck Lever /* Error when copying to the receive buffer, 1203a246b010SChuck Lever * usually because we weren't able to allocate 1204a246b010SChuck Lever * additional buffer pages. All we can do now 1205e136d092SChuck Lever * is turn off TCP_RCV_COPY_DATA, so the request 1206a246b010SChuck Lever * will not receive any additional updates, 1207a246b010SChuck Lever * and time out. 1208a246b010SChuck Lever * Any remaining data from this record will 1209a246b010SChuck Lever * be discarded. 1210a246b010SChuck Lever */ 1211e136d092SChuck Lever transport->tcp_flags &= ~TCP_RCV_COPY_DATA; 1212a246b010SChuck Lever dprintk("RPC: XID %08x truncated request\n", 121351971139SChuck Lever ntohl(transport->tcp_xid)); 121446121cf7SChuck Lever dprintk("RPC: xprt = %p, tcp_copied = %lu, " 121546121cf7SChuck Lever "tcp_offset = %u, tcp_reclen = %u\n", 121646121cf7SChuck Lever xprt, transport->tcp_copied, 121746121cf7SChuck Lever transport->tcp_offset, transport->tcp_reclen); 121844b98efdSRicardo Labiaga return; 1219a246b010SChuck Lever } 1220a246b010SChuck Lever 1221a246b010SChuck Lever dprintk("RPC: XID %08x read %Zd bytes\n", 122251971139SChuck Lever ntohl(transport->tcp_xid), r); 122346121cf7SChuck Lever dprintk("RPC: xprt = %p, tcp_copied = %lu, tcp_offset = %u, " 122446121cf7SChuck Lever "tcp_reclen = %u\n", xprt, transport->tcp_copied, 122546121cf7SChuck Lever transport->tcp_offset, transport->tcp_reclen); 1226a246b010SChuck Lever 122751971139SChuck Lever if (transport->tcp_copied == req->rq_private_buf.buflen) 1228e136d092SChuck Lever transport->tcp_flags &= ~TCP_RCV_COPY_DATA; 122951971139SChuck Lever else if (transport->tcp_offset == transport->tcp_reclen) { 1230e136d092SChuck Lever if (transport->tcp_flags & TCP_RCV_LAST_FRAG) 1231e136d092SChuck Lever transport->tcp_flags &= ~TCP_RCV_COPY_DATA; 1232a246b010SChuck Lever } 123344b98efdSRicardo Labiaga } 123444b98efdSRicardo Labiaga 123544b98efdSRicardo Labiaga /* 123644b98efdSRicardo Labiaga * Finds the request corresponding to the RPC xid and invokes the common 123744b98efdSRicardo Labiaga * tcp read code to read the data. 123844b98efdSRicardo Labiaga */ 123944b98efdSRicardo Labiaga static inline int xs_tcp_read_reply(struct rpc_xprt *xprt, 124044b98efdSRicardo Labiaga struct xdr_skb_reader *desc) 124144b98efdSRicardo Labiaga { 124244b98efdSRicardo Labiaga struct sock_xprt *transport = 124344b98efdSRicardo Labiaga container_of(xprt, struct sock_xprt, xprt); 124444b98efdSRicardo Labiaga struct rpc_rqst *req; 124544b98efdSRicardo Labiaga 124644b98efdSRicardo Labiaga dprintk("RPC: read reply XID %08x\n", ntohl(transport->tcp_xid)); 124744b98efdSRicardo Labiaga 124844b98efdSRicardo Labiaga /* Find and lock the request corresponding to this xid */ 1249*edc1b01cSTrond Myklebust spin_lock_bh(&xprt->transport_lock); 125044b98efdSRicardo Labiaga req = xprt_lookup_rqst(xprt, transport->tcp_xid); 125144b98efdSRicardo Labiaga if (!req) { 125244b98efdSRicardo Labiaga dprintk("RPC: XID %08x request not found!\n", 125344b98efdSRicardo Labiaga ntohl(transport->tcp_xid)); 1254*edc1b01cSTrond Myklebust spin_unlock_bh(&xprt->transport_lock); 125544b98efdSRicardo Labiaga return -1; 125644b98efdSRicardo Labiaga } 125744b98efdSRicardo Labiaga 125844b98efdSRicardo Labiaga xs_tcp_read_common(xprt, desc, req); 125944b98efdSRicardo Labiaga 1260e136d092SChuck Lever if (!(transport->tcp_flags & TCP_RCV_COPY_DATA)) 126151971139SChuck Lever xprt_complete_rqst(req->rq_task, transport->tcp_copied); 126244b98efdSRicardo Labiaga 1263*edc1b01cSTrond Myklebust spin_unlock_bh(&xprt->transport_lock); 126444b98efdSRicardo Labiaga return 0; 126544b98efdSRicardo Labiaga } 126644b98efdSRicardo Labiaga 12679e00abc3STrond Myklebust #if defined(CONFIG_SUNRPC_BACKCHANNEL) 126844b98efdSRicardo Labiaga /* 126944b98efdSRicardo Labiaga * Obtains an rpc_rqst previously allocated and invokes the common 127044b98efdSRicardo Labiaga * tcp read code to read the data. The result is placed in the callback 127144b98efdSRicardo Labiaga * queue. 127244b98efdSRicardo Labiaga * If we're unable to obtain the rpc_rqst we schedule the closing of the 127344b98efdSRicardo Labiaga * connection and return -1. 127444b98efdSRicardo Labiaga */ 12752ea24497STrond Myklebust static int xs_tcp_read_callback(struct rpc_xprt *xprt, 127644b98efdSRicardo Labiaga struct xdr_skb_reader *desc) 127744b98efdSRicardo Labiaga { 127844b98efdSRicardo Labiaga struct sock_xprt *transport = 127944b98efdSRicardo Labiaga container_of(xprt, struct sock_xprt, xprt); 128044b98efdSRicardo Labiaga struct rpc_rqst *req; 128144b98efdSRicardo Labiaga 12822ea24497STrond Myklebust /* Look up and lock the request corresponding to the given XID */ 1283*edc1b01cSTrond Myklebust spin_lock_bh(&xprt->transport_lock); 12842ea24497STrond Myklebust req = xprt_lookup_bc_request(xprt, transport->tcp_xid); 128544b98efdSRicardo Labiaga if (req == NULL) { 1286*edc1b01cSTrond Myklebust spin_unlock_bh(&xprt->transport_lock); 128744b98efdSRicardo Labiaga printk(KERN_WARNING "Callback slot table overflowed\n"); 128844b98efdSRicardo Labiaga xprt_force_disconnect(xprt); 128944b98efdSRicardo Labiaga return -1; 129044b98efdSRicardo Labiaga } 129144b98efdSRicardo Labiaga 129244b98efdSRicardo Labiaga dprintk("RPC: read callback XID %08x\n", ntohl(req->rq_xid)); 129344b98efdSRicardo Labiaga xs_tcp_read_common(xprt, desc, req); 129444b98efdSRicardo Labiaga 12952ea24497STrond Myklebust if (!(transport->tcp_flags & TCP_RCV_COPY_DATA)) 12962ea24497STrond Myklebust xprt_complete_bc_request(req, transport->tcp_copied); 1297*edc1b01cSTrond Myklebust spin_unlock_bh(&xprt->transport_lock); 129844b98efdSRicardo Labiaga 129944b98efdSRicardo Labiaga return 0; 130044b98efdSRicardo Labiaga } 130144b98efdSRicardo Labiaga 130244b98efdSRicardo Labiaga static inline int _xs_tcp_read_data(struct rpc_xprt *xprt, 130344b98efdSRicardo Labiaga struct xdr_skb_reader *desc) 130444b98efdSRicardo Labiaga { 130544b98efdSRicardo Labiaga struct sock_xprt *transport = 130644b98efdSRicardo Labiaga container_of(xprt, struct sock_xprt, xprt); 130744b98efdSRicardo Labiaga 130844b98efdSRicardo Labiaga return (transport->tcp_flags & TCP_RPC_REPLY) ? 130944b98efdSRicardo Labiaga xs_tcp_read_reply(xprt, desc) : 131044b98efdSRicardo Labiaga xs_tcp_read_callback(xprt, desc); 131144b98efdSRicardo Labiaga } 131244b98efdSRicardo Labiaga #else 131344b98efdSRicardo Labiaga static inline int _xs_tcp_read_data(struct rpc_xprt *xprt, 131444b98efdSRicardo Labiaga struct xdr_skb_reader *desc) 131544b98efdSRicardo Labiaga { 131644b98efdSRicardo Labiaga return xs_tcp_read_reply(xprt, desc); 131744b98efdSRicardo Labiaga } 13189e00abc3STrond Myklebust #endif /* CONFIG_SUNRPC_BACKCHANNEL */ 131944b98efdSRicardo Labiaga 132044b98efdSRicardo Labiaga /* 132144b98efdSRicardo Labiaga * Read data off the transport. This can be either an RPC_CALL or an 132244b98efdSRicardo Labiaga * RPC_REPLY. Relay the processing to helper functions. 132344b98efdSRicardo Labiaga */ 132444b98efdSRicardo Labiaga static void xs_tcp_read_data(struct rpc_xprt *xprt, 132544b98efdSRicardo Labiaga struct xdr_skb_reader *desc) 132644b98efdSRicardo Labiaga { 132744b98efdSRicardo Labiaga struct sock_xprt *transport = 132844b98efdSRicardo Labiaga container_of(xprt, struct sock_xprt, xprt); 132944b98efdSRicardo Labiaga 133044b98efdSRicardo Labiaga if (_xs_tcp_read_data(xprt, desc) == 0) 133151971139SChuck Lever xs_tcp_check_fraghdr(transport); 133244b98efdSRicardo Labiaga else { 133344b98efdSRicardo Labiaga /* 133444b98efdSRicardo Labiaga * The transport_lock protects the request handling. 133544b98efdSRicardo Labiaga * There's no need to hold it to update the tcp_flags. 133644b98efdSRicardo Labiaga */ 133744b98efdSRicardo Labiaga transport->tcp_flags &= ~TCP_RCV_COPY_DATA; 133844b98efdSRicardo Labiaga } 1339a246b010SChuck Lever } 1340a246b010SChuck Lever 1341dd456471SChuck Lever static inline void xs_tcp_read_discard(struct sock_xprt *transport, struct xdr_skb_reader *desc) 1342a246b010SChuck Lever { 1343a246b010SChuck Lever size_t len; 1344a246b010SChuck Lever 134551971139SChuck Lever len = transport->tcp_reclen - transport->tcp_offset; 1346a246b010SChuck Lever if (len > desc->count) 1347a246b010SChuck Lever len = desc->count; 1348a246b010SChuck Lever desc->count -= len; 1349a246b010SChuck Lever desc->offset += len; 135051971139SChuck Lever transport->tcp_offset += len; 1351a246b010SChuck Lever dprintk("RPC: discarded %Zu bytes\n", len); 135251971139SChuck Lever xs_tcp_check_fraghdr(transport); 1353a246b010SChuck Lever } 1354a246b010SChuck Lever 13559903cd1cSChuck Lever static int xs_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, unsigned int offset, size_t len) 1356a246b010SChuck Lever { 1357a246b010SChuck Lever struct rpc_xprt *xprt = rd_desc->arg.data; 135851971139SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1359dd456471SChuck Lever struct xdr_skb_reader desc = { 1360a246b010SChuck Lever .skb = skb, 1361a246b010SChuck Lever .offset = offset, 1362a246b010SChuck Lever .count = len, 1363a246b010SChuck Lever }; 1364a246b010SChuck Lever 13659903cd1cSChuck Lever dprintk("RPC: xs_tcp_data_recv started\n"); 1366a246b010SChuck Lever do { 13671a867a08SJeff Layton trace_xs_tcp_data_recv(transport); 1368a246b010SChuck Lever /* Read in a new fragment marker if necessary */ 1369a246b010SChuck Lever /* Can we ever really expect to get completely empty fragments? */ 1370e136d092SChuck Lever if (transport->tcp_flags & TCP_RCV_COPY_FRAGHDR) { 13719903cd1cSChuck Lever xs_tcp_read_fraghdr(xprt, &desc); 1372a246b010SChuck Lever continue; 1373a246b010SChuck Lever } 1374a246b010SChuck Lever /* Read in the xid if necessary */ 1375e136d092SChuck Lever if (transport->tcp_flags & TCP_RCV_COPY_XID) { 137651971139SChuck Lever xs_tcp_read_xid(transport, &desc); 1377a246b010SChuck Lever continue; 1378a246b010SChuck Lever } 137918dca02aSRicardo Labiaga /* Read in the call/reply flag */ 1380f4a2e418SRicardo Labiaga if (transport->tcp_flags & TCP_RCV_READ_CALLDIR) { 138118dca02aSRicardo Labiaga xs_tcp_read_calldir(transport, &desc); 138218dca02aSRicardo Labiaga continue; 138318dca02aSRicardo Labiaga } 1384a246b010SChuck Lever /* Read in the request data */ 1385e136d092SChuck Lever if (transport->tcp_flags & TCP_RCV_COPY_DATA) { 138644b98efdSRicardo Labiaga xs_tcp_read_data(xprt, &desc); 1387a246b010SChuck Lever continue; 1388a246b010SChuck Lever } 1389a246b010SChuck Lever /* Skip over any trailing bytes on short reads */ 139051971139SChuck Lever xs_tcp_read_discard(transport, &desc); 1391a246b010SChuck Lever } while (desc.count); 13921a867a08SJeff Layton trace_xs_tcp_data_recv(transport); 13939903cd1cSChuck Lever dprintk("RPC: xs_tcp_data_recv done\n"); 1394a246b010SChuck Lever return len - desc.count; 1395a246b010SChuck Lever } 1396a246b010SChuck Lever 139766d7a56aSTrond Myklebust static void xs_tcp_data_receive(struct sock_xprt *transport) 139866d7a56aSTrond Myklebust { 139966d7a56aSTrond Myklebust struct rpc_xprt *xprt = &transport->xprt; 140066d7a56aSTrond Myklebust struct sock *sk; 140166d7a56aSTrond Myklebust read_descriptor_t rd_desc = { 140266d7a56aSTrond Myklebust .count = 2*1024*1024, 140366d7a56aSTrond Myklebust .arg.data = xprt, 140466d7a56aSTrond Myklebust }; 140566d7a56aSTrond Myklebust unsigned long total = 0; 140666d7a56aSTrond Myklebust int read = 0; 140766d7a56aSTrond Myklebust 1408*edc1b01cSTrond Myklebust mutex_lock(&transport->recv_mutex); 140966d7a56aSTrond Myklebust sk = transport->inet; 1410*edc1b01cSTrond Myklebust if (sk == NULL) 1411*edc1b01cSTrond Myklebust goto out; 141266d7a56aSTrond Myklebust 141366d7a56aSTrond Myklebust /* We use rd_desc to pass struct xprt to xs_tcp_data_recv */ 141466d7a56aSTrond Myklebust for (;;) { 1415*edc1b01cSTrond Myklebust lock_sock(sk); 141666d7a56aSTrond Myklebust read = tcp_read_sock(sk, &rd_desc, xs_tcp_data_recv); 1417*edc1b01cSTrond Myklebust release_sock(sk); 141866d7a56aSTrond Myklebust if (read <= 0) 141966d7a56aSTrond Myklebust break; 142066d7a56aSTrond Myklebust total += read; 142166d7a56aSTrond Myklebust rd_desc.count = 65536; 142266d7a56aSTrond Myklebust } 1423*edc1b01cSTrond Myklebust out: 1424*edc1b01cSTrond Myklebust mutex_unlock(&transport->recv_mutex); 142566d7a56aSTrond Myklebust trace_xs_tcp_data_ready(xprt, read, total); 142666d7a56aSTrond Myklebust } 142766d7a56aSTrond Myklebust 1428*edc1b01cSTrond Myklebust static void xs_tcp_data_receive_workfn(struct work_struct *work) 1429*edc1b01cSTrond Myklebust { 1430*edc1b01cSTrond Myklebust struct sock_xprt *transport = 1431*edc1b01cSTrond Myklebust container_of(work, struct sock_xprt, recv_worker); 1432*edc1b01cSTrond Myklebust xs_tcp_data_receive(transport); 1433*edc1b01cSTrond Myklebust } 1434*edc1b01cSTrond Myklebust 14359903cd1cSChuck Lever /** 14369903cd1cSChuck Lever * xs_tcp_data_ready - "data ready" callback for TCP sockets 14379903cd1cSChuck Lever * @sk: socket with data to read 14389903cd1cSChuck Lever * 14399903cd1cSChuck Lever */ 1440676d2369SDavid S. Miller static void xs_tcp_data_ready(struct sock *sk) 1441a246b010SChuck Lever { 144266d7a56aSTrond Myklebust struct sock_xprt *transport; 1443a246b010SChuck Lever struct rpc_xprt *xprt; 1444a246b010SChuck Lever 14459903cd1cSChuck Lever dprintk("RPC: xs_tcp_data_ready...\n"); 144646121cf7SChuck Lever 1447f064af1eSEric Dumazet read_lock_bh(&sk->sk_callback_lock); 144866d7a56aSTrond Myklebust if (!(xprt = xprt_from_sock(sk))) 1449a246b010SChuck Lever goto out; 145066d7a56aSTrond Myklebust transport = container_of(xprt, struct sock_xprt, xprt); 145166d7a56aSTrond Myklebust 145261d0a8e6SNeil Brown /* Any data means we had a useful conversation, so 145361d0a8e6SNeil Brown * the we don't need to delay the next reconnect 145461d0a8e6SNeil Brown */ 145561d0a8e6SNeil Brown if (xprt->reestablish_timeout) 145661d0a8e6SNeil Brown xprt->reestablish_timeout = 0; 1457*edc1b01cSTrond Myklebust queue_work(rpciod_workqueue, &transport->recv_worker); 145861d0a8e6SNeil Brown 1459a246b010SChuck Lever out: 1460f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1461a246b010SChuck Lever } 1462a246b010SChuck Lever 14639903cd1cSChuck Lever /** 14649903cd1cSChuck Lever * xs_tcp_state_change - callback to handle TCP socket state changes 14659903cd1cSChuck Lever * @sk: socket whose state has changed 14669903cd1cSChuck Lever * 14679903cd1cSChuck Lever */ 14689903cd1cSChuck Lever static void xs_tcp_state_change(struct sock *sk) 1469a246b010SChuck Lever { 1470a246b010SChuck Lever struct rpc_xprt *xprt; 14710fdea1e8STrond Myklebust struct sock_xprt *transport; 1472a246b010SChuck Lever 1473f064af1eSEric Dumazet read_lock_bh(&sk->sk_callback_lock); 1474a246b010SChuck Lever if (!(xprt = xprt_from_sock(sk))) 1475a246b010SChuck Lever goto out; 14769903cd1cSChuck Lever dprintk("RPC: xs_tcp_state_change client %p...\n", xprt); 1477669502ffSAndy Chittenden dprintk("RPC: state %x conn %d dead %d zapped %d sk_shutdown %d\n", 1478a246b010SChuck Lever sk->sk_state, xprt_connected(xprt), 1479a246b010SChuck Lever sock_flag(sk, SOCK_DEAD), 1480669502ffSAndy Chittenden sock_flag(sk, SOCK_ZAPPED), 1481669502ffSAndy Chittenden sk->sk_shutdown); 1482a246b010SChuck Lever 14830fdea1e8STrond Myklebust transport = container_of(xprt, struct sock_xprt, xprt); 148440b5ea0cSTrond Myklebust trace_rpc_socket_state_change(xprt, sk->sk_socket); 1485a246b010SChuck Lever switch (sk->sk_state) { 1486a246b010SChuck Lever case TCP_ESTABLISHED: 1487f064af1eSEric Dumazet spin_lock(&xprt->transport_lock); 1488a246b010SChuck Lever if (!xprt_test_and_set_connected(xprt)) { 148951971139SChuck Lever 1490a246b010SChuck Lever /* Reset TCP record info */ 149151971139SChuck Lever transport->tcp_offset = 0; 149251971139SChuck Lever transport->tcp_reclen = 0; 149351971139SChuck Lever transport->tcp_copied = 0; 1494e136d092SChuck Lever transport->tcp_flags = 1495e136d092SChuck Lever TCP_RCV_COPY_FRAGHDR | TCP_RCV_COPY_XID; 14968b71798cSTrond Myklebust xprt->connect_cookie++; 14970fdea1e8STrond Myklebust clear_bit(XPRT_SOCK_CONNECTING, &transport->sock_state); 14980fdea1e8STrond Myklebust xprt_clear_connecting(xprt); 149951971139SChuck Lever 15002a491991STrond Myklebust xprt_wake_pending_tasks(xprt, -EAGAIN); 1501a246b010SChuck Lever } 1502f064af1eSEric Dumazet spin_unlock(&xprt->transport_lock); 1503a246b010SChuck Lever break; 15043b948ae5STrond Myklebust case TCP_FIN_WAIT1: 15053b948ae5STrond Myklebust /* The client initiated a shutdown of the socket */ 15067c1d71cfSTrond Myklebust xprt->connect_cookie++; 1507663b8858STrond Myklebust xprt->reestablish_timeout = 0; 15083b948ae5STrond Myklebust set_bit(XPRT_CLOSING, &xprt->state); 15094e857c58SPeter Zijlstra smp_mb__before_atomic(); 15103b948ae5STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state); 1511ef803670STrond Myklebust clear_bit(XPRT_CLOSE_WAIT, &xprt->state); 15124e857c58SPeter Zijlstra smp_mb__after_atomic(); 1513a246b010SChuck Lever break; 1514632e3bdcSTrond Myklebust case TCP_CLOSE_WAIT: 15153b948ae5STrond Myklebust /* The server initiated a shutdown of the socket */ 15167c1d71cfSTrond Myklebust xprt->connect_cookie++; 1517d0bea455STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state); 1518a519fc7aSTrond Myklebust xs_tcp_force_close(xprt); 1519663b8858STrond Myklebust case TCP_CLOSING: 1520663b8858STrond Myklebust /* 1521663b8858STrond Myklebust * If the server closed down the connection, make sure that 1522663b8858STrond Myklebust * we back off before reconnecting 1523663b8858STrond Myklebust */ 1524663b8858STrond Myklebust if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO) 1525663b8858STrond Myklebust xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 15263b948ae5STrond Myklebust break; 15273b948ae5STrond Myklebust case TCP_LAST_ACK: 1528670f9457STrond Myklebust set_bit(XPRT_CLOSING, &xprt->state); 15294e857c58SPeter Zijlstra smp_mb__before_atomic(); 15303b948ae5STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state); 15314e857c58SPeter Zijlstra smp_mb__after_atomic(); 15323b948ae5STrond Myklebust break; 15333b948ae5STrond Myklebust case TCP_CLOSE: 15340fdea1e8STrond Myklebust if (test_and_clear_bit(XPRT_SOCK_CONNECTING, 15350fdea1e8STrond Myklebust &transport->sock_state)) 15360fdea1e8STrond Myklebust xprt_clear_connecting(xprt); 15377d1e8255STrond Myklebust xs_sock_mark_closed(xprt); 1538a246b010SChuck Lever } 1539a246b010SChuck Lever out: 1540f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1541a246b010SChuck Lever } 1542a246b010SChuck Lever 15431f0fa154SIlpo Järvinen static void xs_write_space(struct sock *sk) 15441f0fa154SIlpo Järvinen { 15451f0fa154SIlpo Järvinen struct socket *sock; 15461f0fa154SIlpo Järvinen struct rpc_xprt *xprt; 15471f0fa154SIlpo Järvinen 15481f0fa154SIlpo Järvinen if (unlikely(!(sock = sk->sk_socket))) 15491f0fa154SIlpo Järvinen return; 15501f0fa154SIlpo Järvinen clear_bit(SOCK_NOSPACE, &sock->flags); 15511f0fa154SIlpo Järvinen 15521f0fa154SIlpo Järvinen if (unlikely(!(xprt = xprt_from_sock(sk)))) 15531f0fa154SIlpo Järvinen return; 15541f0fa154SIlpo Järvinen if (test_and_clear_bit(SOCK_ASYNC_NOSPACE, &sock->flags) == 0) 15551f0fa154SIlpo Järvinen return; 15561f0fa154SIlpo Järvinen 15571f0fa154SIlpo Järvinen xprt_write_space(xprt); 15581f0fa154SIlpo Järvinen } 15591f0fa154SIlpo Järvinen 15602a9e1cfaSTrond Myklebust /** 1561c7b2cae8SChuck Lever * xs_udp_write_space - callback invoked when socket buffer space 1562c7b2cae8SChuck Lever * becomes available 15639903cd1cSChuck Lever * @sk: socket whose state has changed 15649903cd1cSChuck Lever * 1565a246b010SChuck Lever * Called when more output buffer space is available for this socket. 1566a246b010SChuck Lever * We try not to wake our writers until they can make "significant" 1567c7b2cae8SChuck Lever * progress, otherwise we'll waste resources thrashing kernel_sendmsg 1568a246b010SChuck Lever * with a bunch of small requests. 1569a246b010SChuck Lever */ 1570c7b2cae8SChuck Lever static void xs_udp_write_space(struct sock *sk) 1571a246b010SChuck Lever { 1572f064af1eSEric Dumazet read_lock_bh(&sk->sk_callback_lock); 1573c7b2cae8SChuck Lever 1574c7b2cae8SChuck Lever /* from net/core/sock.c:sock_def_write_space */ 15751f0fa154SIlpo Järvinen if (sock_writeable(sk)) 15761f0fa154SIlpo Järvinen xs_write_space(sk); 1577c7b2cae8SChuck Lever 1578f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1579c7b2cae8SChuck Lever } 1580c7b2cae8SChuck Lever 1581c7b2cae8SChuck Lever /** 1582c7b2cae8SChuck Lever * xs_tcp_write_space - callback invoked when socket buffer space 1583c7b2cae8SChuck Lever * becomes available 1584c7b2cae8SChuck Lever * @sk: socket whose state has changed 1585c7b2cae8SChuck Lever * 1586c7b2cae8SChuck Lever * Called when more output buffer space is available for this socket. 1587c7b2cae8SChuck Lever * We try not to wake our writers until they can make "significant" 1588c7b2cae8SChuck Lever * progress, otherwise we'll waste resources thrashing kernel_sendmsg 1589c7b2cae8SChuck Lever * with a bunch of small requests. 1590c7b2cae8SChuck Lever */ 1591c7b2cae8SChuck Lever static void xs_tcp_write_space(struct sock *sk) 1592c7b2cae8SChuck Lever { 1593f064af1eSEric Dumazet read_lock_bh(&sk->sk_callback_lock); 1594c7b2cae8SChuck Lever 1595c7b2cae8SChuck Lever /* from net/core/stream.c:sk_stream_write_space */ 159664dc6130SEric Dumazet if (sk_stream_is_writeable(sk)) 15971f0fa154SIlpo Järvinen xs_write_space(sk); 1598c7b2cae8SChuck Lever 1599f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1600a246b010SChuck Lever } 1601a246b010SChuck Lever 1602470056c2SChuck Lever static void xs_udp_do_set_buffer_size(struct rpc_xprt *xprt) 1603a246b010SChuck Lever { 1604ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1605ee0ac0c2SChuck Lever struct sock *sk = transport->inet; 1606a246b010SChuck Lever 16077c6e066eSChuck Lever if (transport->rcvsize) { 1608a246b010SChuck Lever sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 16097c6e066eSChuck Lever sk->sk_rcvbuf = transport->rcvsize * xprt->max_reqs * 2; 1610a246b010SChuck Lever } 16117c6e066eSChuck Lever if (transport->sndsize) { 1612a246b010SChuck Lever sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 16137c6e066eSChuck Lever sk->sk_sndbuf = transport->sndsize * xprt->max_reqs * 2; 1614a246b010SChuck Lever sk->sk_write_space(sk); 1615a246b010SChuck Lever } 1616a246b010SChuck Lever } 1617a246b010SChuck Lever 161843118c29SChuck Lever /** 1619470056c2SChuck Lever * xs_udp_set_buffer_size - set send and receive limits 162043118c29SChuck Lever * @xprt: generic transport 1621470056c2SChuck Lever * @sndsize: requested size of send buffer, in bytes 1622470056c2SChuck Lever * @rcvsize: requested size of receive buffer, in bytes 162343118c29SChuck Lever * 1624470056c2SChuck Lever * Set socket send and receive buffer size limits. 162543118c29SChuck Lever */ 1626470056c2SChuck Lever static void xs_udp_set_buffer_size(struct rpc_xprt *xprt, size_t sndsize, size_t rcvsize) 162743118c29SChuck Lever { 16287c6e066eSChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 16297c6e066eSChuck Lever 16307c6e066eSChuck Lever transport->sndsize = 0; 1631470056c2SChuck Lever if (sndsize) 16327c6e066eSChuck Lever transport->sndsize = sndsize + 1024; 16337c6e066eSChuck Lever transport->rcvsize = 0; 1634470056c2SChuck Lever if (rcvsize) 16357c6e066eSChuck Lever transport->rcvsize = rcvsize + 1024; 1636470056c2SChuck Lever 1637470056c2SChuck Lever xs_udp_do_set_buffer_size(xprt); 163843118c29SChuck Lever } 163943118c29SChuck Lever 164046c0ee8bSChuck Lever /** 164146c0ee8bSChuck Lever * xs_udp_timer - called when a retransmit timeout occurs on a UDP transport 164246c0ee8bSChuck Lever * @task: task that timed out 164346c0ee8bSChuck Lever * 164446c0ee8bSChuck Lever * Adjust the congestion window after a retransmit timeout has occurred. 164546c0ee8bSChuck Lever */ 16466a24dfb6STrond Myklebust static void xs_udp_timer(struct rpc_xprt *xprt, struct rpc_task *task) 164746c0ee8bSChuck Lever { 16486a24dfb6STrond Myklebust xprt_adjust_cwnd(xprt, task, -ETIMEDOUT); 164946c0ee8bSChuck Lever } 165046c0ee8bSChuck Lever 1651b85d8806SChuck Lever static unsigned short xs_get_random_port(void) 1652b85d8806SChuck Lever { 1653b85d8806SChuck Lever unsigned short range = xprt_max_resvport - xprt_min_resvport; 165463862b5bSAruna-Hewapathirane unsigned short rand = (unsigned short) prandom_u32() % range; 1655b85d8806SChuck Lever return rand + xprt_min_resvport; 1656b85d8806SChuck Lever } 1657b85d8806SChuck Lever 165892200412SChuck Lever /** 16594dda9c8aSTrond Myklebust * xs_set_reuseaddr_port - set the socket's port and address reuse options 16604dda9c8aSTrond Myklebust * @sock: socket 16614dda9c8aSTrond Myklebust * 16624dda9c8aSTrond Myklebust * Note that this function has to be called on all sockets that share the 16634dda9c8aSTrond Myklebust * same port, and it must be called before binding. 16644dda9c8aSTrond Myklebust */ 16654dda9c8aSTrond Myklebust static void xs_sock_set_reuseport(struct socket *sock) 16664dda9c8aSTrond Myklebust { 1667402e23b4STrond Myklebust int opt = 1; 16684dda9c8aSTrond Myklebust 1669402e23b4STrond Myklebust kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, 1670402e23b4STrond Myklebust (char *)&opt, sizeof(opt)); 16714dda9c8aSTrond Myklebust } 16724dda9c8aSTrond Myklebust 16734dda9c8aSTrond Myklebust static unsigned short xs_sock_getport(struct socket *sock) 16744dda9c8aSTrond Myklebust { 16754dda9c8aSTrond Myklebust struct sockaddr_storage buf; 16764dda9c8aSTrond Myklebust int buflen; 16774dda9c8aSTrond Myklebust unsigned short port = 0; 16784dda9c8aSTrond Myklebust 16794dda9c8aSTrond Myklebust if (kernel_getsockname(sock, (struct sockaddr *)&buf, &buflen) < 0) 16804dda9c8aSTrond Myklebust goto out; 16814dda9c8aSTrond Myklebust switch (buf.ss_family) { 16824dda9c8aSTrond Myklebust case AF_INET6: 16834dda9c8aSTrond Myklebust port = ntohs(((struct sockaddr_in6 *)&buf)->sin6_port); 16844dda9c8aSTrond Myklebust break; 16854dda9c8aSTrond Myklebust case AF_INET: 16864dda9c8aSTrond Myklebust port = ntohs(((struct sockaddr_in *)&buf)->sin_port); 16874dda9c8aSTrond Myklebust } 16884dda9c8aSTrond Myklebust out: 16894dda9c8aSTrond Myklebust return port; 16904dda9c8aSTrond Myklebust } 16914dda9c8aSTrond Myklebust 16924dda9c8aSTrond Myklebust /** 169392200412SChuck Lever * xs_set_port - reset the port number in the remote endpoint address 169492200412SChuck Lever * @xprt: generic transport 169592200412SChuck Lever * @port: new port number 169692200412SChuck Lever * 169792200412SChuck Lever */ 169892200412SChuck Lever static void xs_set_port(struct rpc_xprt *xprt, unsigned short port) 169992200412SChuck Lever { 170092200412SChuck Lever dprintk("RPC: setting port for xprt %p to %u\n", xprt, port); 1701c4efcb1dSChuck Lever 17029dc3b095SChuck Lever rpc_set_port(xs_addr(xprt), port); 17039dc3b095SChuck Lever xs_update_peer_port(xprt); 170492200412SChuck Lever } 170592200412SChuck Lever 17064dda9c8aSTrond Myklebust static void xs_set_srcport(struct sock_xprt *transport, struct socket *sock) 17074dda9c8aSTrond Myklebust { 17084dda9c8aSTrond Myklebust if (transport->srcport == 0) 17094dda9c8aSTrond Myklebust transport->srcport = xs_sock_getport(sock); 17104dda9c8aSTrond Myklebust } 17114dda9c8aSTrond Myklebust 17125d4ec932SPavel Emelyanov static unsigned short xs_get_srcport(struct sock_xprt *transport) 171367a391d7STrond Myklebust { 1714fbfffbd5SChuck Lever unsigned short port = transport->srcport; 171567a391d7STrond Myklebust 171667a391d7STrond Myklebust if (port == 0 && transport->xprt.resvport) 171767a391d7STrond Myklebust port = xs_get_random_port(); 171867a391d7STrond Myklebust return port; 171967a391d7STrond Myklebust } 172067a391d7STrond Myklebust 1721baaf4e48SPavel Emelyanov static unsigned short xs_next_srcport(struct sock_xprt *transport, unsigned short port) 172267a391d7STrond Myklebust { 1723fbfffbd5SChuck Lever if (transport->srcport != 0) 1724fbfffbd5SChuck Lever transport->srcport = 0; 172567a391d7STrond Myklebust if (!transport->xprt.resvport) 172667a391d7STrond Myklebust return 0; 172767a391d7STrond Myklebust if (port <= xprt_min_resvport || port > xprt_max_resvport) 172867a391d7STrond Myklebust return xprt_max_resvport; 172967a391d7STrond Myklebust return --port; 173067a391d7STrond Myklebust } 1731beb59b68SPavel Emelyanov static int xs_bind(struct sock_xprt *transport, struct socket *sock) 1732a246b010SChuck Lever { 1733beb59b68SPavel Emelyanov struct sockaddr_storage myaddr; 173467a391d7STrond Myklebust int err, nloop = 0; 17355d4ec932SPavel Emelyanov unsigned short port = xs_get_srcport(transport); 173667a391d7STrond Myklebust unsigned short last; 1737a246b010SChuck Lever 17380f7a622cSChris Perl /* 17390f7a622cSChris Perl * If we are asking for any ephemeral port (i.e. port == 0 && 17400f7a622cSChris Perl * transport->xprt.resvport == 0), don't bind. Let the local 17410f7a622cSChris Perl * port selection happen implicitly when the socket is used 17420f7a622cSChris Perl * (for example at connect time). 17430f7a622cSChris Perl * 17440f7a622cSChris Perl * This ensures that we can continue to establish TCP 17450f7a622cSChris Perl * connections even when all local ephemeral ports are already 17460f7a622cSChris Perl * a part of some TCP connection. This makes no difference 17470f7a622cSChris Perl * for UDP sockets, but also doens't harm them. 17480f7a622cSChris Perl * 17490f7a622cSChris Perl * If we're asking for any reserved port (i.e. port == 0 && 17500f7a622cSChris Perl * transport->xprt.resvport == 1) xs_get_srcport above will 17510f7a622cSChris Perl * ensure that port is non-zero and we will bind as needed. 17520f7a622cSChris Perl */ 17530f7a622cSChris Perl if (port == 0) 17540f7a622cSChris Perl return 0; 17550f7a622cSChris Perl 1756beb59b68SPavel Emelyanov memcpy(&myaddr, &transport->srcaddr, transport->xprt.addrlen); 1757a246b010SChuck Lever do { 1758beb59b68SPavel Emelyanov rpc_set_port((struct sockaddr *)&myaddr, port); 1759e6242e92SSridhar Samudrala err = kernel_bind(sock, (struct sockaddr *)&myaddr, 1760beb59b68SPavel Emelyanov transport->xprt.addrlen); 1761a246b010SChuck Lever if (err == 0) { 1762fbfffbd5SChuck Lever transport->srcport = port; 1763d3bc9a1dSFrank van Maarseveen break; 1764a246b010SChuck Lever } 176567a391d7STrond Myklebust last = port; 1766baaf4e48SPavel Emelyanov port = xs_next_srcport(transport, port); 176767a391d7STrond Myklebust if (port > last) 176867a391d7STrond Myklebust nloop++; 176967a391d7STrond Myklebust } while (err == -EADDRINUSE && nloop != 2); 1770beb59b68SPavel Emelyanov 17714232e863SChuck Lever if (myaddr.ss_family == AF_INET) 1772beb59b68SPavel Emelyanov dprintk("RPC: %s %pI4:%u: %s (%d)\n", __func__, 1773beb59b68SPavel Emelyanov &((struct sockaddr_in *)&myaddr)->sin_addr, 1774beb59b68SPavel Emelyanov port, err ? "failed" : "ok", err); 1775beb59b68SPavel Emelyanov else 1776beb59b68SPavel Emelyanov dprintk("RPC: %s %pI6:%u: %s (%d)\n", __func__, 1777beb59b68SPavel Emelyanov &((struct sockaddr_in6 *)&myaddr)->sin6_addr, 17787dc753f0SChuck Lever port, err ? "failed" : "ok", err); 1779a246b010SChuck Lever return err; 1780a246b010SChuck Lever } 1781a246b010SChuck Lever 1782176e21eeSChuck Lever /* 1783176e21eeSChuck Lever * We don't support autobind on AF_LOCAL sockets 1784176e21eeSChuck Lever */ 1785176e21eeSChuck Lever static void xs_local_rpcbind(struct rpc_task *task) 1786176e21eeSChuck Lever { 17873dc0da27STrond Myklebust rcu_read_lock(); 17883dc0da27STrond Myklebust xprt_set_bound(rcu_dereference(task->tk_client->cl_xprt)); 17893dc0da27STrond Myklebust rcu_read_unlock(); 1790176e21eeSChuck Lever } 1791176e21eeSChuck Lever 1792176e21eeSChuck Lever static void xs_local_set_port(struct rpc_xprt *xprt, unsigned short port) 1793176e21eeSChuck Lever { 1794176e21eeSChuck Lever } 1795a246b010SChuck Lever 1796ed07536eSPeter Zijlstra #ifdef CONFIG_DEBUG_LOCK_ALLOC 1797ed07536eSPeter Zijlstra static struct lock_class_key xs_key[2]; 1798ed07536eSPeter Zijlstra static struct lock_class_key xs_slock_key[2]; 1799ed07536eSPeter Zijlstra 1800176e21eeSChuck Lever static inline void xs_reclassify_socketu(struct socket *sock) 1801176e21eeSChuck Lever { 1802176e21eeSChuck Lever struct sock *sk = sock->sk; 1803176e21eeSChuck Lever 1804176e21eeSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_LOCAL-RPC", 1805176e21eeSChuck Lever &xs_slock_key[1], "sk_lock-AF_LOCAL-RPC", &xs_key[1]); 1806176e21eeSChuck Lever } 1807176e21eeSChuck Lever 18088945ee5eSChuck Lever static inline void xs_reclassify_socket4(struct socket *sock) 1809ed07536eSPeter Zijlstra { 1810ed07536eSPeter Zijlstra struct sock *sk = sock->sk; 18118945ee5eSChuck Lever 18128945ee5eSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_INET-RPC", 18138945ee5eSChuck Lever &xs_slock_key[0], "sk_lock-AF_INET-RPC", &xs_key[0]); 1814ed07536eSPeter Zijlstra } 18158945ee5eSChuck Lever 18168945ee5eSChuck Lever static inline void xs_reclassify_socket6(struct socket *sock) 18178945ee5eSChuck Lever { 18188945ee5eSChuck Lever struct sock *sk = sock->sk; 18198945ee5eSChuck Lever 18208945ee5eSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_INET6-RPC", 18218945ee5eSChuck Lever &xs_slock_key[1], "sk_lock-AF_INET6-RPC", &xs_key[1]); 1822ed07536eSPeter Zijlstra } 18236bc9638aSPavel Emelyanov 18246bc9638aSPavel Emelyanov static inline void xs_reclassify_socket(int family, struct socket *sock) 18256bc9638aSPavel Emelyanov { 18261b7a1819SWeston Andros Adamson WARN_ON_ONCE(sock_owned_by_user(sock->sk)); 18271b7a1819SWeston Andros Adamson if (sock_owned_by_user(sock->sk)) 18281b7a1819SWeston Andros Adamson return; 18291b7a1819SWeston Andros Adamson 18304232e863SChuck Lever switch (family) { 1831176e21eeSChuck Lever case AF_LOCAL: 1832176e21eeSChuck Lever xs_reclassify_socketu(sock); 1833176e21eeSChuck Lever break; 18344232e863SChuck Lever case AF_INET: 18356bc9638aSPavel Emelyanov xs_reclassify_socket4(sock); 18364232e863SChuck Lever break; 18374232e863SChuck Lever case AF_INET6: 18386bc9638aSPavel Emelyanov xs_reclassify_socket6(sock); 18394232e863SChuck Lever break; 18404232e863SChuck Lever } 18416bc9638aSPavel Emelyanov } 1842ed07536eSPeter Zijlstra #else 1843176e21eeSChuck Lever static inline void xs_reclassify_socketu(struct socket *sock) 1844176e21eeSChuck Lever { 1845176e21eeSChuck Lever } 1846176e21eeSChuck Lever 18478945ee5eSChuck Lever static inline void xs_reclassify_socket4(struct socket *sock) 18488945ee5eSChuck Lever { 18498945ee5eSChuck Lever } 18508945ee5eSChuck Lever 18518945ee5eSChuck Lever static inline void xs_reclassify_socket6(struct socket *sock) 1852ed07536eSPeter Zijlstra { 1853ed07536eSPeter Zijlstra } 18546bc9638aSPavel Emelyanov 18556bc9638aSPavel Emelyanov static inline void xs_reclassify_socket(int family, struct socket *sock) 18566bc9638aSPavel Emelyanov { 18576bc9638aSPavel Emelyanov } 1858ed07536eSPeter Zijlstra #endif 1859ed07536eSPeter Zijlstra 1860*edc1b01cSTrond Myklebust static void xs_dummy_data_receive_workfn(struct work_struct *work) 1861*edc1b01cSTrond Myklebust { 1862*edc1b01cSTrond Myklebust } 1863*edc1b01cSTrond Myklebust 186493dc41bdSNeilBrown static void xs_dummy_setup_socket(struct work_struct *work) 186593dc41bdSNeilBrown { 186693dc41bdSNeilBrown } 186793dc41bdSNeilBrown 18686bc9638aSPavel Emelyanov static struct socket *xs_create_sock(struct rpc_xprt *xprt, 18694dda9c8aSTrond Myklebust struct sock_xprt *transport, int family, int type, 18704dda9c8aSTrond Myklebust int protocol, bool reuseport) 187122f79326SPavel Emelyanov { 187222f79326SPavel Emelyanov struct socket *sock; 187322f79326SPavel Emelyanov int err; 187422f79326SPavel Emelyanov 18756bc9638aSPavel Emelyanov err = __sock_create(xprt->xprt_net, family, type, protocol, &sock, 1); 187622f79326SPavel Emelyanov if (err < 0) { 187722f79326SPavel Emelyanov dprintk("RPC: can't create %d transport socket (%d).\n", 187822f79326SPavel Emelyanov protocol, -err); 187922f79326SPavel Emelyanov goto out; 188022f79326SPavel Emelyanov } 18816bc9638aSPavel Emelyanov xs_reclassify_socket(family, sock); 188222f79326SPavel Emelyanov 18834dda9c8aSTrond Myklebust if (reuseport) 18844dda9c8aSTrond Myklebust xs_sock_set_reuseport(sock); 18854dda9c8aSTrond Myklebust 18864cea288aSBen Hutchings err = xs_bind(transport, sock); 18874cea288aSBen Hutchings if (err) { 188822f79326SPavel Emelyanov sock_release(sock); 188922f79326SPavel Emelyanov goto out; 189022f79326SPavel Emelyanov } 189122f79326SPavel Emelyanov 189222f79326SPavel Emelyanov return sock; 189322f79326SPavel Emelyanov out: 189422f79326SPavel Emelyanov return ERR_PTR(err); 189522f79326SPavel Emelyanov } 189622f79326SPavel Emelyanov 1897176e21eeSChuck Lever static int xs_local_finish_connecting(struct rpc_xprt *xprt, 1898176e21eeSChuck Lever struct socket *sock) 1899176e21eeSChuck Lever { 1900176e21eeSChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, 1901176e21eeSChuck Lever xprt); 1902176e21eeSChuck Lever 1903176e21eeSChuck Lever if (!transport->inet) { 1904176e21eeSChuck Lever struct sock *sk = sock->sk; 1905176e21eeSChuck Lever 1906176e21eeSChuck Lever write_lock_bh(&sk->sk_callback_lock); 1907176e21eeSChuck Lever 1908176e21eeSChuck Lever xs_save_old_callbacks(transport, sk); 1909176e21eeSChuck Lever 1910176e21eeSChuck Lever sk->sk_user_data = xprt; 1911176e21eeSChuck Lever sk->sk_data_ready = xs_local_data_ready; 1912176e21eeSChuck Lever sk->sk_write_space = xs_udp_write_space; 19132118071dSTrond Myklebust sk->sk_error_report = xs_error_report; 1914c2126157STrond Myklebust sk->sk_allocation = GFP_NOIO; 1915176e21eeSChuck Lever 1916176e21eeSChuck Lever xprt_clear_connected(xprt); 1917176e21eeSChuck Lever 1918176e21eeSChuck Lever /* Reset to new socket */ 1919176e21eeSChuck Lever transport->sock = sock; 1920176e21eeSChuck Lever transport->inet = sk; 1921176e21eeSChuck Lever 1922176e21eeSChuck Lever write_unlock_bh(&sk->sk_callback_lock); 1923176e21eeSChuck Lever } 1924176e21eeSChuck Lever 1925176e21eeSChuck Lever /* Tell the socket layer to start connecting... */ 1926176e21eeSChuck Lever xprt->stat.connect_count++; 1927176e21eeSChuck Lever xprt->stat.connect_start = jiffies; 1928176e21eeSChuck Lever return kernel_connect(sock, xs_addr(xprt), xprt->addrlen, 0); 1929176e21eeSChuck Lever } 1930176e21eeSChuck Lever 1931176e21eeSChuck Lever /** 1932176e21eeSChuck Lever * xs_local_setup_socket - create AF_LOCAL socket, connect to a local endpoint 1933176e21eeSChuck Lever * @transport: socket transport to connect 1934176e21eeSChuck Lever */ 1935dc107402SJ. Bruce Fields static int xs_local_setup_socket(struct sock_xprt *transport) 1936176e21eeSChuck Lever { 1937176e21eeSChuck Lever struct rpc_xprt *xprt = &transport->xprt; 1938176e21eeSChuck Lever struct socket *sock; 1939176e21eeSChuck Lever int status = -EIO; 1940176e21eeSChuck Lever 1941176e21eeSChuck Lever status = __sock_create(xprt->xprt_net, AF_LOCAL, 1942176e21eeSChuck Lever SOCK_STREAM, 0, &sock, 1); 1943176e21eeSChuck Lever if (status < 0) { 1944176e21eeSChuck Lever dprintk("RPC: can't create AF_LOCAL " 1945176e21eeSChuck Lever "transport socket (%d).\n", -status); 1946176e21eeSChuck Lever goto out; 1947176e21eeSChuck Lever } 1948176e21eeSChuck Lever xs_reclassify_socketu(sock); 1949176e21eeSChuck Lever 1950176e21eeSChuck Lever dprintk("RPC: worker connecting xprt %p via AF_LOCAL to %s\n", 1951176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 1952176e21eeSChuck Lever 1953176e21eeSChuck Lever status = xs_local_finish_connecting(xprt, sock); 195440b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, status); 1955176e21eeSChuck Lever switch (status) { 1956176e21eeSChuck Lever case 0: 1957176e21eeSChuck Lever dprintk("RPC: xprt %p connected to %s\n", 1958176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 1959176e21eeSChuck Lever xprt_set_connected(xprt); 19603601c4a9STrond Myklebust case -ENOBUFS: 1961176e21eeSChuck Lever break; 1962176e21eeSChuck Lever case -ENOENT: 1963176e21eeSChuck Lever dprintk("RPC: xprt %p: socket %s does not exist\n", 1964176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 1965176e21eeSChuck Lever break; 19664a20a988STrond Myklebust case -ECONNREFUSED: 19674a20a988STrond Myklebust dprintk("RPC: xprt %p: connection refused for %s\n", 19684a20a988STrond Myklebust xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 19694a20a988STrond Myklebust break; 1970176e21eeSChuck Lever default: 1971176e21eeSChuck Lever printk(KERN_ERR "%s: unhandled error (%d) connecting to %s\n", 1972176e21eeSChuck Lever __func__, -status, 1973176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR]); 1974176e21eeSChuck Lever } 1975176e21eeSChuck Lever 1976176e21eeSChuck Lever out: 1977176e21eeSChuck Lever xprt_clear_connecting(xprt); 1978176e21eeSChuck Lever xprt_wake_pending_tasks(xprt, status); 1979dc107402SJ. Bruce Fields return status; 1980dc107402SJ. Bruce Fields } 1981dc107402SJ. Bruce Fields 1982b6669737SLinus Torvalds static void xs_local_connect(struct rpc_xprt *xprt, struct rpc_task *task) 1983dc107402SJ. Bruce Fields { 1984dc107402SJ. Bruce Fields struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1985dc107402SJ. Bruce Fields int ret; 1986dc107402SJ. Bruce Fields 1987dc107402SJ. Bruce Fields if (RPC_IS_ASYNC(task)) { 1988dc107402SJ. Bruce Fields /* 1989dc107402SJ. Bruce Fields * We want the AF_LOCAL connect to be resolved in the 1990dc107402SJ. Bruce Fields * filesystem namespace of the process making the rpc 1991dc107402SJ. Bruce Fields * call. Thus we connect synchronously. 1992dc107402SJ. Bruce Fields * 1993dc107402SJ. Bruce Fields * If we want to support asynchronous AF_LOCAL calls, 1994dc107402SJ. Bruce Fields * we'll need to figure out how to pass a namespace to 1995dc107402SJ. Bruce Fields * connect. 1996dc107402SJ. Bruce Fields */ 1997dc107402SJ. Bruce Fields rpc_exit(task, -ENOTCONN); 1998dc107402SJ. Bruce Fields return; 1999dc107402SJ. Bruce Fields } 2000dc107402SJ. Bruce Fields ret = xs_local_setup_socket(transport); 2001dc107402SJ. Bruce Fields if (ret && !RPC_IS_SOFTCONN(task)) 2002dc107402SJ. Bruce Fields msleep_interruptible(15000); 2003176e21eeSChuck Lever } 2004176e21eeSChuck Lever 20053c87ef6eSJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_SWAP) 2006d6e971d8SJeff Layton /* 2007d6e971d8SJeff Layton * Note that this should be called with XPRT_LOCKED held (or when we otherwise 2008d6e971d8SJeff Layton * know that we have exclusive access to the socket), to guard against 2009d6e971d8SJeff Layton * races with xs_reset_transport. 2010d6e971d8SJeff Layton */ 2011a564b8f0SMel Gorman static void xs_set_memalloc(struct rpc_xprt *xprt) 2012a564b8f0SMel Gorman { 2013a564b8f0SMel Gorman struct sock_xprt *transport = container_of(xprt, struct sock_xprt, 2014a564b8f0SMel Gorman xprt); 2015a564b8f0SMel Gorman 2016d6e971d8SJeff Layton /* 2017d6e971d8SJeff Layton * If there's no sock, then we have nothing to set. The 2018d6e971d8SJeff Layton * reconnecting process will get it for us. 2019d6e971d8SJeff Layton */ 2020d6e971d8SJeff Layton if (!transport->inet) 2021d6e971d8SJeff Layton return; 20228e228133SJeff Layton if (atomic_read(&xprt->swapper)) 2023a564b8f0SMel Gorman sk_set_memalloc(transport->inet); 2024a564b8f0SMel Gorman } 2025a564b8f0SMel Gorman 2026a564b8f0SMel Gorman /** 2027d67fa4d8SJeff Layton * xs_enable_swap - Tag this transport as being used for swap. 2028a564b8f0SMel Gorman * @xprt: transport to tag 2029a564b8f0SMel Gorman * 20308e228133SJeff Layton * Take a reference to this transport on behalf of the rpc_clnt, and 20318e228133SJeff Layton * optionally mark it for swapping if it wasn't already. 2032a564b8f0SMel Gorman */ 2033d67fa4d8SJeff Layton static int 2034d67fa4d8SJeff Layton xs_enable_swap(struct rpc_xprt *xprt) 2035a564b8f0SMel Gorman { 2036d6e971d8SJeff Layton struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt); 2037a564b8f0SMel Gorman 2038d6e971d8SJeff Layton if (atomic_inc_return(&xprt->swapper) != 1) 2039d6e971d8SJeff Layton return 0; 2040d6e971d8SJeff Layton if (wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE)) 2041d6e971d8SJeff Layton return -ERESTARTSYS; 2042d6e971d8SJeff Layton if (xs->inet) 2043d6e971d8SJeff Layton sk_set_memalloc(xs->inet); 2044d6e971d8SJeff Layton xprt_release_xprt(xprt, NULL); 20458e228133SJeff Layton return 0; 2046a564b8f0SMel Gorman } 2047a564b8f0SMel Gorman 20488e228133SJeff Layton /** 2049d67fa4d8SJeff Layton * xs_disable_swap - Untag this transport as being used for swap. 20508e228133SJeff Layton * @xprt: transport to tag 20518e228133SJeff Layton * 20528e228133SJeff Layton * Drop a "swapper" reference to this xprt on behalf of the rpc_clnt. If the 20538e228133SJeff Layton * swapper refcount goes to 0, untag the socket as a memalloc socket. 20548e228133SJeff Layton */ 2055d67fa4d8SJeff Layton static void 2056d67fa4d8SJeff Layton xs_disable_swap(struct rpc_xprt *xprt) 20578e228133SJeff Layton { 2058d6e971d8SJeff Layton struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt); 20598e228133SJeff Layton 2060d6e971d8SJeff Layton if (!atomic_dec_and_test(&xprt->swapper)) 2061d6e971d8SJeff Layton return; 2062d6e971d8SJeff Layton if (wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE)) 2063d6e971d8SJeff Layton return; 2064d6e971d8SJeff Layton if (xs->inet) 2065d6e971d8SJeff Layton sk_clear_memalloc(xs->inet); 2066d6e971d8SJeff Layton xprt_release_xprt(xprt, NULL); 2067a564b8f0SMel Gorman } 2068a564b8f0SMel Gorman #else 2069a564b8f0SMel Gorman static void xs_set_memalloc(struct rpc_xprt *xprt) 2070a564b8f0SMel Gorman { 2071a564b8f0SMel Gorman } 2072d67fa4d8SJeff Layton 2073d67fa4d8SJeff Layton static int 2074d67fa4d8SJeff Layton xs_enable_swap(struct rpc_xprt *xprt) 2075d67fa4d8SJeff Layton { 2076d67fa4d8SJeff Layton return -EINVAL; 2077d67fa4d8SJeff Layton } 2078d67fa4d8SJeff Layton 2079d67fa4d8SJeff Layton static void 2080d67fa4d8SJeff Layton xs_disable_swap(struct rpc_xprt *xprt) 2081d67fa4d8SJeff Layton { 2082d67fa4d8SJeff Layton } 2083a564b8f0SMel Gorman #endif 2084a564b8f0SMel Gorman 208516be2d20SChuck Lever static void xs_udp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock) 2086a246b010SChuck Lever { 208716be2d20SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2088edb267a6SChuck Lever 2089ee0ac0c2SChuck Lever if (!transport->inet) { 2090b0d93ad5SChuck Lever struct sock *sk = sock->sk; 2091b0d93ad5SChuck Lever 2092b0d93ad5SChuck Lever write_lock_bh(&sk->sk_callback_lock); 2093b0d93ad5SChuck Lever 20942a9e1cfaSTrond Myklebust xs_save_old_callbacks(transport, sk); 20952a9e1cfaSTrond Myklebust 2096b0d93ad5SChuck Lever sk->sk_user_data = xprt; 2097b0d93ad5SChuck Lever sk->sk_data_ready = xs_udp_data_ready; 2098b0d93ad5SChuck Lever sk->sk_write_space = xs_udp_write_space; 2099c2126157STrond Myklebust sk->sk_allocation = GFP_NOIO; 2100b0d93ad5SChuck Lever 2101b0d93ad5SChuck Lever xprt_set_connected(xprt); 2102b0d93ad5SChuck Lever 2103b0d93ad5SChuck Lever /* Reset to new socket */ 2104ee0ac0c2SChuck Lever transport->sock = sock; 2105ee0ac0c2SChuck Lever transport->inet = sk; 2106b0d93ad5SChuck Lever 2107a564b8f0SMel Gorman xs_set_memalloc(xprt); 2108a564b8f0SMel Gorman 2109b0d93ad5SChuck Lever write_unlock_bh(&sk->sk_callback_lock); 2110b0d93ad5SChuck Lever } 2111470056c2SChuck Lever xs_udp_do_set_buffer_size(xprt); 211216be2d20SChuck Lever } 211316be2d20SChuck Lever 21148c14ff2aSPavel Emelyanov static void xs_udp_setup_socket(struct work_struct *work) 2115a246b010SChuck Lever { 2116a246b010SChuck Lever struct sock_xprt *transport = 2117a246b010SChuck Lever container_of(work, struct sock_xprt, connect_worker.work); 2118a246b010SChuck Lever struct rpc_xprt *xprt = &transport->xprt; 2119a246b010SChuck Lever struct socket *sock = transport->sock; 2120b65c0310SPavel Emelyanov int status = -EIO; 2121a246b010SChuck Lever 21228c14ff2aSPavel Emelyanov sock = xs_create_sock(xprt, transport, 21234dda9c8aSTrond Myklebust xs_addr(xprt)->sa_family, SOCK_DGRAM, 21244dda9c8aSTrond Myklebust IPPROTO_UDP, false); 2125b65c0310SPavel Emelyanov if (IS_ERR(sock)) 2126a246b010SChuck Lever goto out; 212768e220bdSChuck Lever 2128c740eff8SChuck Lever dprintk("RPC: worker connecting xprt %p via %s to " 2129c740eff8SChuck Lever "%s (port %s)\n", xprt, 2130c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO], 2131c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2132c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT]); 213368e220bdSChuck Lever 213468e220bdSChuck Lever xs_udp_finish_connecting(xprt, sock); 213540b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, 0); 2136a246b010SChuck Lever status = 0; 2137b0d93ad5SChuck Lever out: 2138718ba5b8STrond Myklebust xprt_unlock_connect(xprt, transport); 2139b0d93ad5SChuck Lever xprt_clear_connecting(xprt); 21407d1e8255STrond Myklebust xprt_wake_pending_tasks(xprt, status); 2141b0d93ad5SChuck Lever } 2142b0d93ad5SChuck Lever 21434876cc77STrond Myklebust /** 21444876cc77STrond Myklebust * xs_tcp_shutdown - gracefully shut down a TCP socket 21454876cc77STrond Myklebust * @xprt: transport 21464876cc77STrond Myklebust * 21474876cc77STrond Myklebust * Initiates a graceful shutdown of the TCP socket by calling the 21484876cc77STrond Myklebust * equivalent of shutdown(SHUT_RDWR); 21494876cc77STrond Myklebust */ 21504876cc77STrond Myklebust static void xs_tcp_shutdown(struct rpc_xprt *xprt) 21514876cc77STrond Myklebust { 21524876cc77STrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 21534876cc77STrond Myklebust struct socket *sock = transport->sock; 21544876cc77STrond Myklebust 21554876cc77STrond Myklebust if (sock == NULL) 21564876cc77STrond Myklebust return; 21574876cc77STrond Myklebust if (xprt_connected(xprt)) { 21584876cc77STrond Myklebust kernel_sock_shutdown(sock, SHUT_RDWR); 21594876cc77STrond Myklebust trace_rpc_socket_shutdown(xprt, sock); 21604876cc77STrond Myklebust } else 21614876cc77STrond Myklebust xs_reset_transport(transport); 21624876cc77STrond Myklebust } 21634876cc77STrond Myklebust 216416be2d20SChuck Lever static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock) 2165b0d93ad5SChuck Lever { 216616be2d20SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2167fe19a96bSTrond Myklebust int ret = -ENOTCONN; 2168edb267a6SChuck Lever 2169ee0ac0c2SChuck Lever if (!transport->inet) { 2170b0d93ad5SChuck Lever struct sock *sk = sock->sk; 21717f260e85STrond Myklebust unsigned int keepidle = xprt->timeout->to_initval / HZ; 21727f260e85STrond Myklebust unsigned int keepcnt = xprt->timeout->to_retries + 1; 21737f260e85STrond Myklebust unsigned int opt_on = 1; 2174775f06abSTrond Myklebust unsigned int timeo; 21757f260e85STrond Myklebust 21767f260e85STrond Myklebust /* TCP Keepalive options */ 21777f260e85STrond Myklebust kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, 21787f260e85STrond Myklebust (char *)&opt_on, sizeof(opt_on)); 21797f260e85STrond Myklebust kernel_setsockopt(sock, SOL_TCP, TCP_KEEPIDLE, 21807f260e85STrond Myklebust (char *)&keepidle, sizeof(keepidle)); 21817f260e85STrond Myklebust kernel_setsockopt(sock, SOL_TCP, TCP_KEEPINTVL, 21827f260e85STrond Myklebust (char *)&keepidle, sizeof(keepidle)); 21837f260e85STrond Myklebust kernel_setsockopt(sock, SOL_TCP, TCP_KEEPCNT, 21847f260e85STrond Myklebust (char *)&keepcnt, sizeof(keepcnt)); 2185b0d93ad5SChuck Lever 2186775f06abSTrond Myklebust /* TCP user timeout (see RFC5482) */ 2187775f06abSTrond Myklebust timeo = jiffies_to_msecs(xprt->timeout->to_initval) * 2188775f06abSTrond Myklebust (xprt->timeout->to_retries + 1); 2189775f06abSTrond Myklebust kernel_setsockopt(sock, SOL_TCP, TCP_USER_TIMEOUT, 2190775f06abSTrond Myklebust (char *)&timeo, sizeof(timeo)); 2191775f06abSTrond Myklebust 2192b0d93ad5SChuck Lever write_lock_bh(&sk->sk_callback_lock); 2193b0d93ad5SChuck Lever 21942a9e1cfaSTrond Myklebust xs_save_old_callbacks(transport, sk); 21952a9e1cfaSTrond Myklebust 2196b0d93ad5SChuck Lever sk->sk_user_data = xprt; 2197b0d93ad5SChuck Lever sk->sk_data_ready = xs_tcp_data_ready; 2198b0d93ad5SChuck Lever sk->sk_state_change = xs_tcp_state_change; 2199b0d93ad5SChuck Lever sk->sk_write_space = xs_tcp_write_space; 22002118071dSTrond Myklebust sk->sk_error_report = xs_error_report; 2201c2126157STrond Myklebust sk->sk_allocation = GFP_NOIO; 22023167e12cSChuck Lever 22033167e12cSChuck Lever /* socket options */ 22043167e12cSChuck Lever sock_reset_flag(sk, SOCK_LINGER); 22053167e12cSChuck Lever tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF; 2206b0d93ad5SChuck Lever 2207b0d93ad5SChuck Lever xprt_clear_connected(xprt); 2208b0d93ad5SChuck Lever 2209b0d93ad5SChuck Lever /* Reset to new socket */ 2210ee0ac0c2SChuck Lever transport->sock = sock; 2211ee0ac0c2SChuck Lever transport->inet = sk; 2212b0d93ad5SChuck Lever 2213b0d93ad5SChuck Lever write_unlock_bh(&sk->sk_callback_lock); 2214b0d93ad5SChuck Lever } 2215b0d93ad5SChuck Lever 221601d37c42STrond Myklebust if (!xprt_bound(xprt)) 2217fe19a96bSTrond Myklebust goto out; 221801d37c42STrond Myklebust 2219a564b8f0SMel Gorman xs_set_memalloc(xprt); 2220a564b8f0SMel Gorman 2221b0d93ad5SChuck Lever /* Tell the socket layer to start connecting... */ 2222262ca07dSChuck Lever xprt->stat.connect_count++; 2223262ca07dSChuck Lever xprt->stat.connect_start = jiffies; 22240fdea1e8STrond Myklebust set_bit(XPRT_SOCK_CONNECTING, &transport->sock_state); 2225fe19a96bSTrond Myklebust ret = kernel_connect(sock, xs_addr(xprt), xprt->addrlen, O_NONBLOCK); 2226fe19a96bSTrond Myklebust switch (ret) { 2227fe19a96bSTrond Myklebust case 0: 22284dda9c8aSTrond Myklebust xs_set_srcport(transport, sock); 2229fe19a96bSTrond Myklebust case -EINPROGRESS: 2230fe19a96bSTrond Myklebust /* SYN_SENT! */ 2231fe19a96bSTrond Myklebust if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO) 2232fe19a96bSTrond Myklebust xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 2233fe19a96bSTrond Myklebust } 2234fe19a96bSTrond Myklebust out: 2235fe19a96bSTrond Myklebust return ret; 223616be2d20SChuck Lever } 223716be2d20SChuck Lever 223816be2d20SChuck Lever /** 2239b61d59ffSTrond Myklebust * xs_tcp_setup_socket - create a TCP socket and connect to a remote endpoint 224016be2d20SChuck Lever * 224116be2d20SChuck Lever * Invoked by a work queue tasklet. 224216be2d20SChuck Lever */ 2243cdd518d5SPavel Emelyanov static void xs_tcp_setup_socket(struct work_struct *work) 224416be2d20SChuck Lever { 2245cdd518d5SPavel Emelyanov struct sock_xprt *transport = 2246cdd518d5SPavel Emelyanov container_of(work, struct sock_xprt, connect_worker.work); 224716be2d20SChuck Lever struct socket *sock = transport->sock; 2248a9f5f0f7SPavel Emelyanov struct rpc_xprt *xprt = &transport->xprt; 2249b61d59ffSTrond Myklebust int status = -EIO; 225016be2d20SChuck Lever 225116be2d20SChuck Lever if (!sock) { 2252cdd518d5SPavel Emelyanov sock = xs_create_sock(xprt, transport, 22534dda9c8aSTrond Myklebust xs_addr(xprt)->sa_family, SOCK_STREAM, 22544dda9c8aSTrond Myklebust IPPROTO_TCP, true); 2255b61d59ffSTrond Myklebust if (IS_ERR(sock)) { 2256b61d59ffSTrond Myklebust status = PTR_ERR(sock); 225716be2d20SChuck Lever goto out; 225816be2d20SChuck Lever } 22597d1e8255STrond Myklebust } 22607d1e8255STrond Myklebust 2261c740eff8SChuck Lever dprintk("RPC: worker connecting xprt %p via %s to " 2262c740eff8SChuck Lever "%s (port %s)\n", xprt, 2263c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO], 2264c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2265c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT]); 226616be2d20SChuck Lever 226716be2d20SChuck Lever status = xs_tcp_finish_connecting(xprt, sock); 226840b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, status); 2269a246b010SChuck Lever dprintk("RPC: %p connect status %d connected %d sock state %d\n", 227046121cf7SChuck Lever xprt, -status, xprt_connected(xprt), 227146121cf7SChuck Lever sock->sk->sk_state); 2272a246b010SChuck Lever switch (status) { 2273f75e6745STrond Myklebust default: 2274f75e6745STrond Myklebust printk("%s: connect returned unhandled error %d\n", 2275f75e6745STrond Myklebust __func__, status); 2276f75e6745STrond Myklebust case -EADDRNOTAVAIL: 2277f75e6745STrond Myklebust /* We're probably in TIME_WAIT. Get rid of existing socket, 2278f75e6745STrond Myklebust * and retry 2279f75e6745STrond Myklebust */ 2280a519fc7aSTrond Myklebust xs_tcp_force_close(xprt); 228188b5ed73STrond Myklebust break; 22822a491991STrond Myklebust case 0: 2283a246b010SChuck Lever case -EINPROGRESS: 2284a246b010SChuck Lever case -EALREADY: 2285718ba5b8STrond Myklebust xprt_unlock_connect(xprt, transport); 22867d1e8255STrond Myklebust return; 22879fcfe0c8STrond Myklebust case -EINVAL: 22889fcfe0c8STrond Myklebust /* Happens, for instance, if the user specified a link 22899fcfe0c8STrond Myklebust * local IPv6 address without a scope-id. 22909fcfe0c8STrond Myklebust */ 22913ed5e2a2STrond Myklebust case -ECONNREFUSED: 22923ed5e2a2STrond Myklebust case -ECONNRESET: 22933ed5e2a2STrond Myklebust case -ENETUNREACH: 22943913c78cSTrond Myklebust case -EADDRINUSE: 22953601c4a9STrond Myklebust case -ENOBUFS: 22963ed5e2a2STrond Myklebust /* retry with existing socket, after a delay */ 22974efdd92cSTrond Myklebust xs_tcp_force_close(xprt); 22989fcfe0c8STrond Myklebust goto out; 22998a2cec29STrond Myklebust } 23002a491991STrond Myklebust status = -EAGAIN; 2301a246b010SChuck Lever out: 2302718ba5b8STrond Myklebust xprt_unlock_connect(xprt, transport); 23032226feb6SChuck Lever xprt_clear_connecting(xprt); 23047d1e8255STrond Myklebust xprt_wake_pending_tasks(xprt, status); 2305a246b010SChuck Lever } 2306a246b010SChuck Lever 230768e220bdSChuck Lever /** 23089903cd1cSChuck Lever * xs_connect - connect a socket to a remote endpoint 23091b092092STrond Myklebust * @xprt: pointer to transport structure 23109903cd1cSChuck Lever * @task: address of RPC task that manages state of connect request 23119903cd1cSChuck Lever * 23129903cd1cSChuck Lever * TCP: If the remote end dropped the connection, delay reconnecting. 231303bf4b70SChuck Lever * 231403bf4b70SChuck Lever * UDP socket connects are synchronous, but we use a work queue anyway 231503bf4b70SChuck Lever * to guarantee that even unprivileged user processes can set up a 231603bf4b70SChuck Lever * socket on a privileged port. 231703bf4b70SChuck Lever * 231803bf4b70SChuck Lever * If a UDP socket connect fails, the delay behavior here prevents 231903bf4b70SChuck Lever * retry floods (hard mounts). 23209903cd1cSChuck Lever */ 23211b092092STrond Myklebust static void xs_connect(struct rpc_xprt *xprt, struct rpc_task *task) 2322a246b010SChuck Lever { 2323ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2324a246b010SChuck Lever 2325718ba5b8STrond Myklebust WARN_ON_ONCE(!xprt_lock_connect(xprt, task, transport)); 2326718ba5b8STrond Myklebust 232799b1a4c3STrond Myklebust if (transport->sock != NULL) { 232846121cf7SChuck Lever dprintk("RPC: xs_connect delayed xprt %p for %lu " 232946121cf7SChuck Lever "seconds\n", 233003bf4b70SChuck Lever xprt, xprt->reestablish_timeout / HZ); 233199b1a4c3STrond Myklebust 233299b1a4c3STrond Myklebust /* Start by resetting any existing state */ 233399b1a4c3STrond Myklebust xs_reset_transport(transport); 233499b1a4c3STrond Myklebust 2335c1384c9cSTrond Myklebust queue_delayed_work(rpciod_workqueue, 2336c1384c9cSTrond Myklebust &transport->connect_worker, 233703bf4b70SChuck Lever xprt->reestablish_timeout); 233803bf4b70SChuck Lever xprt->reestablish_timeout <<= 1; 233961d0a8e6SNeil Brown if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO) 234061d0a8e6SNeil Brown xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 234103bf4b70SChuck Lever if (xprt->reestablish_timeout > XS_TCP_MAX_REEST_TO) 234203bf4b70SChuck Lever xprt->reestablish_timeout = XS_TCP_MAX_REEST_TO; 23439903cd1cSChuck Lever } else { 23449903cd1cSChuck Lever dprintk("RPC: xs_connect scheduled xprt %p\n", xprt); 2345c1384c9cSTrond Myklebust queue_delayed_work(rpciod_workqueue, 2346c1384c9cSTrond Myklebust &transport->connect_worker, 0); 2347a246b010SChuck Lever } 2348a246b010SChuck Lever } 2349a246b010SChuck Lever 2350262ca07dSChuck Lever /** 2351176e21eeSChuck Lever * xs_local_print_stats - display AF_LOCAL socket-specifc stats 2352176e21eeSChuck Lever * @xprt: rpc_xprt struct containing statistics 2353176e21eeSChuck Lever * @seq: output file 2354176e21eeSChuck Lever * 2355176e21eeSChuck Lever */ 2356176e21eeSChuck Lever static void xs_local_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) 2357176e21eeSChuck Lever { 2358176e21eeSChuck Lever long idle_time = 0; 2359176e21eeSChuck Lever 2360176e21eeSChuck Lever if (xprt_connected(xprt)) 2361176e21eeSChuck Lever idle_time = (long)(jiffies - xprt->last_used) / HZ; 2362176e21eeSChuck Lever 2363176e21eeSChuck Lever seq_printf(seq, "\txprt:\tlocal %lu %lu %lu %ld %lu %lu %lu " 236415a45206SAndy Adamson "%llu %llu %lu %llu %llu\n", 2365176e21eeSChuck Lever xprt->stat.bind_count, 2366176e21eeSChuck Lever xprt->stat.connect_count, 2367176e21eeSChuck Lever xprt->stat.connect_time, 2368176e21eeSChuck Lever idle_time, 2369176e21eeSChuck Lever xprt->stat.sends, 2370176e21eeSChuck Lever xprt->stat.recvs, 2371176e21eeSChuck Lever xprt->stat.bad_xids, 2372176e21eeSChuck Lever xprt->stat.req_u, 237315a45206SAndy Adamson xprt->stat.bklog_u, 237415a45206SAndy Adamson xprt->stat.max_slots, 237515a45206SAndy Adamson xprt->stat.sending_u, 237615a45206SAndy Adamson xprt->stat.pending_u); 2377176e21eeSChuck Lever } 2378176e21eeSChuck Lever 2379176e21eeSChuck Lever /** 2380262ca07dSChuck Lever * xs_udp_print_stats - display UDP socket-specifc stats 2381262ca07dSChuck Lever * @xprt: rpc_xprt struct containing statistics 2382262ca07dSChuck Lever * @seq: output file 2383262ca07dSChuck Lever * 2384262ca07dSChuck Lever */ 2385262ca07dSChuck Lever static void xs_udp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) 2386262ca07dSChuck Lever { 2387c8475461SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2388c8475461SChuck Lever 238915a45206SAndy Adamson seq_printf(seq, "\txprt:\tudp %u %lu %lu %lu %lu %llu %llu " 239015a45206SAndy Adamson "%lu %llu %llu\n", 2391fbfffbd5SChuck Lever transport->srcport, 2392262ca07dSChuck Lever xprt->stat.bind_count, 2393262ca07dSChuck Lever xprt->stat.sends, 2394262ca07dSChuck Lever xprt->stat.recvs, 2395262ca07dSChuck Lever xprt->stat.bad_xids, 2396262ca07dSChuck Lever xprt->stat.req_u, 239715a45206SAndy Adamson xprt->stat.bklog_u, 239815a45206SAndy Adamson xprt->stat.max_slots, 239915a45206SAndy Adamson xprt->stat.sending_u, 240015a45206SAndy Adamson xprt->stat.pending_u); 2401262ca07dSChuck Lever } 2402262ca07dSChuck Lever 2403262ca07dSChuck Lever /** 2404262ca07dSChuck Lever * xs_tcp_print_stats - display TCP socket-specifc stats 2405262ca07dSChuck Lever * @xprt: rpc_xprt struct containing statistics 2406262ca07dSChuck Lever * @seq: output file 2407262ca07dSChuck Lever * 2408262ca07dSChuck Lever */ 2409262ca07dSChuck Lever static void xs_tcp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) 2410262ca07dSChuck Lever { 2411c8475461SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2412262ca07dSChuck Lever long idle_time = 0; 2413262ca07dSChuck Lever 2414262ca07dSChuck Lever if (xprt_connected(xprt)) 2415262ca07dSChuck Lever idle_time = (long)(jiffies - xprt->last_used) / HZ; 2416262ca07dSChuck Lever 241715a45206SAndy Adamson seq_printf(seq, "\txprt:\ttcp %u %lu %lu %lu %ld %lu %lu %lu " 241815a45206SAndy Adamson "%llu %llu %lu %llu %llu\n", 2419fbfffbd5SChuck Lever transport->srcport, 2420262ca07dSChuck Lever xprt->stat.bind_count, 2421262ca07dSChuck Lever xprt->stat.connect_count, 2422262ca07dSChuck Lever xprt->stat.connect_time, 2423262ca07dSChuck Lever idle_time, 2424262ca07dSChuck Lever xprt->stat.sends, 2425262ca07dSChuck Lever xprt->stat.recvs, 2426262ca07dSChuck Lever xprt->stat.bad_xids, 2427262ca07dSChuck Lever xprt->stat.req_u, 242815a45206SAndy Adamson xprt->stat.bklog_u, 242915a45206SAndy Adamson xprt->stat.max_slots, 243015a45206SAndy Adamson xprt->stat.sending_u, 243115a45206SAndy Adamson xprt->stat.pending_u); 2432262ca07dSChuck Lever } 2433262ca07dSChuck Lever 24344cfc7e60SRahul Iyer /* 24354cfc7e60SRahul Iyer * Allocate a bunch of pages for a scratch buffer for the rpc code. The reason 24364cfc7e60SRahul Iyer * we allocate pages instead doing a kmalloc like rpc_malloc is because we want 24374cfc7e60SRahul Iyer * to use the server side send routines. 24384cfc7e60SRahul Iyer */ 24395a51f13aSH Hartley Sweeten static void *bc_malloc(struct rpc_task *task, size_t size) 24404cfc7e60SRahul Iyer { 24414cfc7e60SRahul Iyer struct page *page; 24424cfc7e60SRahul Iyer struct rpc_buffer *buf; 24434cfc7e60SRahul Iyer 2444b8a13d03SWeston Andros Adamson WARN_ON_ONCE(size > PAGE_SIZE - sizeof(struct rpc_buffer)); 2445b8a13d03SWeston Andros Adamson if (size > PAGE_SIZE - sizeof(struct rpc_buffer)) 2446b8a13d03SWeston Andros Adamson return NULL; 24474cfc7e60SRahul Iyer 2448b8a13d03SWeston Andros Adamson page = alloc_page(GFP_KERNEL); 24494cfc7e60SRahul Iyer if (!page) 24504cfc7e60SRahul Iyer return NULL; 24514cfc7e60SRahul Iyer 24524cfc7e60SRahul Iyer buf = page_address(page); 24534cfc7e60SRahul Iyer buf->len = PAGE_SIZE; 24544cfc7e60SRahul Iyer 24554cfc7e60SRahul Iyer return buf->data; 24564cfc7e60SRahul Iyer } 24574cfc7e60SRahul Iyer 24584cfc7e60SRahul Iyer /* 24594cfc7e60SRahul Iyer * Free the space allocated in the bc_alloc routine 24604cfc7e60SRahul Iyer */ 24615a51f13aSH Hartley Sweeten static void bc_free(void *buffer) 24624cfc7e60SRahul Iyer { 24634cfc7e60SRahul Iyer struct rpc_buffer *buf; 24644cfc7e60SRahul Iyer 24654cfc7e60SRahul Iyer if (!buffer) 24664cfc7e60SRahul Iyer return; 24674cfc7e60SRahul Iyer 24684cfc7e60SRahul Iyer buf = container_of(buffer, struct rpc_buffer, data); 24694cfc7e60SRahul Iyer free_page((unsigned long)buf); 24704cfc7e60SRahul Iyer } 24714cfc7e60SRahul Iyer 24724cfc7e60SRahul Iyer /* 24734cfc7e60SRahul Iyer * Use the svc_sock to send the callback. Must be called with svsk->sk_mutex 24744cfc7e60SRahul Iyer * held. Borrows heavily from svc_tcp_sendto and xs_tcp_send_request. 24754cfc7e60SRahul Iyer */ 24764cfc7e60SRahul Iyer static int bc_sendto(struct rpc_rqst *req) 24774cfc7e60SRahul Iyer { 24784cfc7e60SRahul Iyer int len; 24794cfc7e60SRahul Iyer struct xdr_buf *xbufp = &req->rq_snd_buf; 24804cfc7e60SRahul Iyer struct rpc_xprt *xprt = req->rq_xprt; 24814cfc7e60SRahul Iyer struct sock_xprt *transport = 24824cfc7e60SRahul Iyer container_of(xprt, struct sock_xprt, xprt); 24834cfc7e60SRahul Iyer struct socket *sock = transport->sock; 24844cfc7e60SRahul Iyer unsigned long headoff; 24854cfc7e60SRahul Iyer unsigned long tailoff; 24864cfc7e60SRahul Iyer 248761677eeeSChuck Lever xs_encode_stream_record_marker(xbufp); 24884cfc7e60SRahul Iyer 24894cfc7e60SRahul Iyer tailoff = (unsigned long)xbufp->tail[0].iov_base & ~PAGE_MASK; 24904cfc7e60SRahul Iyer headoff = (unsigned long)xbufp->head[0].iov_base & ~PAGE_MASK; 24914cfc7e60SRahul Iyer len = svc_send_common(sock, xbufp, 24924cfc7e60SRahul Iyer virt_to_page(xbufp->head[0].iov_base), headoff, 24934cfc7e60SRahul Iyer xbufp->tail[0].iov_base, tailoff); 24944cfc7e60SRahul Iyer 24954cfc7e60SRahul Iyer if (len != xbufp->len) { 24964cfc7e60SRahul Iyer printk(KERN_NOTICE "Error sending entire callback!\n"); 24974cfc7e60SRahul Iyer len = -EAGAIN; 24984cfc7e60SRahul Iyer } 24994cfc7e60SRahul Iyer 25004cfc7e60SRahul Iyer return len; 25014cfc7e60SRahul Iyer } 25024cfc7e60SRahul Iyer 25034cfc7e60SRahul Iyer /* 25044cfc7e60SRahul Iyer * The send routine. Borrows from svc_send 25054cfc7e60SRahul Iyer */ 25064cfc7e60SRahul Iyer static int bc_send_request(struct rpc_task *task) 25074cfc7e60SRahul Iyer { 25084cfc7e60SRahul Iyer struct rpc_rqst *req = task->tk_rqstp; 25094cfc7e60SRahul Iyer struct svc_xprt *xprt; 25104cfc7e60SRahul Iyer u32 len; 25114cfc7e60SRahul Iyer 25124cfc7e60SRahul Iyer dprintk("sending request with xid: %08x\n", ntohl(req->rq_xid)); 25134cfc7e60SRahul Iyer /* 25144cfc7e60SRahul Iyer * Get the server socket associated with this callback xprt 25154cfc7e60SRahul Iyer */ 25164cfc7e60SRahul Iyer xprt = req->rq_xprt->bc_xprt; 25174cfc7e60SRahul Iyer 25184cfc7e60SRahul Iyer /* 25194cfc7e60SRahul Iyer * Grab the mutex to serialize data as the connection is shared 25204cfc7e60SRahul Iyer * with the fore channel 25214cfc7e60SRahul Iyer */ 25224cfc7e60SRahul Iyer if (!mutex_trylock(&xprt->xpt_mutex)) { 25234cfc7e60SRahul Iyer rpc_sleep_on(&xprt->xpt_bc_pending, task, NULL); 25244cfc7e60SRahul Iyer if (!mutex_trylock(&xprt->xpt_mutex)) 25254cfc7e60SRahul Iyer return -EAGAIN; 25264cfc7e60SRahul Iyer rpc_wake_up_queued_task(&xprt->xpt_bc_pending, task); 25274cfc7e60SRahul Iyer } 25284cfc7e60SRahul Iyer if (test_bit(XPT_DEAD, &xprt->xpt_flags)) 25294cfc7e60SRahul Iyer len = -ENOTCONN; 25304cfc7e60SRahul Iyer else 25314cfc7e60SRahul Iyer len = bc_sendto(req); 25324cfc7e60SRahul Iyer mutex_unlock(&xprt->xpt_mutex); 25334cfc7e60SRahul Iyer 25344cfc7e60SRahul Iyer if (len > 0) 25354cfc7e60SRahul Iyer len = 0; 25364cfc7e60SRahul Iyer 25374cfc7e60SRahul Iyer return len; 25384cfc7e60SRahul Iyer } 25394cfc7e60SRahul Iyer 25404cfc7e60SRahul Iyer /* 25414cfc7e60SRahul Iyer * The close routine. Since this is client initiated, we do nothing 25424cfc7e60SRahul Iyer */ 25434cfc7e60SRahul Iyer 25444cfc7e60SRahul Iyer static void bc_close(struct rpc_xprt *xprt) 25454cfc7e60SRahul Iyer { 25464cfc7e60SRahul Iyer } 25474cfc7e60SRahul Iyer 25484cfc7e60SRahul Iyer /* 25494cfc7e60SRahul Iyer * The xprt destroy routine. Again, because this connection is client 25504cfc7e60SRahul Iyer * initiated, we do nothing 25514cfc7e60SRahul Iyer */ 25524cfc7e60SRahul Iyer 25534cfc7e60SRahul Iyer static void bc_destroy(struct rpc_xprt *xprt) 25544cfc7e60SRahul Iyer { 255547f72efaSKinglong Mee dprintk("RPC: bc_destroy xprt %p\n", xprt); 255647f72efaSKinglong Mee 255747f72efaSKinglong Mee xs_xprt_free(xprt); 255847f72efaSKinglong Mee module_put(THIS_MODULE); 25594cfc7e60SRahul Iyer } 25604cfc7e60SRahul Iyer 2561176e21eeSChuck Lever static struct rpc_xprt_ops xs_local_ops = { 2562176e21eeSChuck Lever .reserve_xprt = xprt_reserve_xprt, 2563176e21eeSChuck Lever .release_xprt = xs_tcp_release_xprt, 2564f39c1bfbSTrond Myklebust .alloc_slot = xprt_alloc_slot, 2565176e21eeSChuck Lever .rpcbind = xs_local_rpcbind, 2566176e21eeSChuck Lever .set_port = xs_local_set_port, 2567dc107402SJ. Bruce Fields .connect = xs_local_connect, 2568176e21eeSChuck Lever .buf_alloc = rpc_malloc, 2569176e21eeSChuck Lever .buf_free = rpc_free, 2570176e21eeSChuck Lever .send_request = xs_local_send_request, 2571176e21eeSChuck Lever .set_retrans_timeout = xprt_set_retrans_timeout_def, 2572176e21eeSChuck Lever .close = xs_close, 2573a1311d87STrond Myklebust .destroy = xs_destroy, 2574176e21eeSChuck Lever .print_stats = xs_local_print_stats, 2575d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2576d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 2577176e21eeSChuck Lever }; 2578176e21eeSChuck Lever 2579262965f5SChuck Lever static struct rpc_xprt_ops xs_udp_ops = { 258043118c29SChuck Lever .set_buffer_size = xs_udp_set_buffer_size, 258112a80469SChuck Lever .reserve_xprt = xprt_reserve_xprt_cong, 258249e9a890SChuck Lever .release_xprt = xprt_release_xprt_cong, 2583f39c1bfbSTrond Myklebust .alloc_slot = xprt_alloc_slot, 258445160d62SChuck Lever .rpcbind = rpcb_getport_async, 258592200412SChuck Lever .set_port = xs_set_port, 25869903cd1cSChuck Lever .connect = xs_connect, 258702107148SChuck Lever .buf_alloc = rpc_malloc, 258802107148SChuck Lever .buf_free = rpc_free, 2589262965f5SChuck Lever .send_request = xs_udp_send_request, 2590fe3aca29SChuck Lever .set_retrans_timeout = xprt_set_retrans_timeout_rtt, 259146c0ee8bSChuck Lever .timer = xs_udp_timer, 2592a58dd398SChuck Lever .release_request = xprt_release_rqst_cong, 2593262965f5SChuck Lever .close = xs_close, 2594262965f5SChuck Lever .destroy = xs_destroy, 2595262ca07dSChuck Lever .print_stats = xs_udp_print_stats, 2596d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2597d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 25984a068258SChuck Lever .inject_disconnect = xs_inject_disconnect, 2599262965f5SChuck Lever }; 2600262965f5SChuck Lever 2601262965f5SChuck Lever static struct rpc_xprt_ops xs_tcp_ops = { 260212a80469SChuck Lever .reserve_xprt = xprt_reserve_xprt, 2603e0ab53deSTrond Myklebust .release_xprt = xs_tcp_release_xprt, 2604f39c1bfbSTrond Myklebust .alloc_slot = xprt_lock_and_alloc_slot, 260545160d62SChuck Lever .rpcbind = rpcb_getport_async, 260692200412SChuck Lever .set_port = xs_set_port, 26070b9e7943STrond Myklebust .connect = xs_connect, 260802107148SChuck Lever .buf_alloc = rpc_malloc, 260902107148SChuck Lever .buf_free = rpc_free, 2610262965f5SChuck Lever .send_request = xs_tcp_send_request, 2611fe3aca29SChuck Lever .set_retrans_timeout = xprt_set_retrans_timeout_def, 2612c627d31bSTrond Myklebust .close = xs_tcp_shutdown, 26139903cd1cSChuck Lever .destroy = xs_destroy, 2614262ca07dSChuck Lever .print_stats = xs_tcp_print_stats, 2615d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2616d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 26174a068258SChuck Lever .inject_disconnect = xs_inject_disconnect, 2618a246b010SChuck Lever }; 2619a246b010SChuck Lever 26204cfc7e60SRahul Iyer /* 26214cfc7e60SRahul Iyer * The rpc_xprt_ops for the server backchannel 26224cfc7e60SRahul Iyer */ 26234cfc7e60SRahul Iyer 26244cfc7e60SRahul Iyer static struct rpc_xprt_ops bc_tcp_ops = { 26254cfc7e60SRahul Iyer .reserve_xprt = xprt_reserve_xprt, 26264cfc7e60SRahul Iyer .release_xprt = xprt_release_xprt, 262784e28a30SBryan Schumaker .alloc_slot = xprt_alloc_slot, 26284cfc7e60SRahul Iyer .buf_alloc = bc_malloc, 26294cfc7e60SRahul Iyer .buf_free = bc_free, 26304cfc7e60SRahul Iyer .send_request = bc_send_request, 26314cfc7e60SRahul Iyer .set_retrans_timeout = xprt_set_retrans_timeout_def, 26324cfc7e60SRahul Iyer .close = bc_close, 26334cfc7e60SRahul Iyer .destroy = bc_destroy, 26344cfc7e60SRahul Iyer .print_stats = xs_tcp_print_stats, 2635d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2636d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 26374a068258SChuck Lever .inject_disconnect = xs_inject_disconnect, 26384cfc7e60SRahul Iyer }; 26394cfc7e60SRahul Iyer 264092476850SChuck Lever static int xs_init_anyaddr(const int family, struct sockaddr *sap) 264192476850SChuck Lever { 264292476850SChuck Lever static const struct sockaddr_in sin = { 264392476850SChuck Lever .sin_family = AF_INET, 264492476850SChuck Lever .sin_addr.s_addr = htonl(INADDR_ANY), 264592476850SChuck Lever }; 264692476850SChuck Lever static const struct sockaddr_in6 sin6 = { 264792476850SChuck Lever .sin6_family = AF_INET6, 264892476850SChuck Lever .sin6_addr = IN6ADDR_ANY_INIT, 264992476850SChuck Lever }; 265092476850SChuck Lever 265192476850SChuck Lever switch (family) { 2652176e21eeSChuck Lever case AF_LOCAL: 2653176e21eeSChuck Lever break; 265492476850SChuck Lever case AF_INET: 265592476850SChuck Lever memcpy(sap, &sin, sizeof(sin)); 265692476850SChuck Lever break; 265792476850SChuck Lever case AF_INET6: 265892476850SChuck Lever memcpy(sap, &sin6, sizeof(sin6)); 265992476850SChuck Lever break; 266092476850SChuck Lever default: 266192476850SChuck Lever dprintk("RPC: %s: Bad address family\n", __func__); 266292476850SChuck Lever return -EAFNOSUPPORT; 266392476850SChuck Lever } 266492476850SChuck Lever return 0; 266592476850SChuck Lever } 266692476850SChuck Lever 26673c341b0bS\"Talpey, Thomas\ static struct rpc_xprt *xs_setup_xprt(struct xprt_create *args, 2668d9ba131dSTrond Myklebust unsigned int slot_table_size, 2669d9ba131dSTrond Myklebust unsigned int max_slot_table_size) 2670c8541ecdSChuck Lever { 2671c8541ecdSChuck Lever struct rpc_xprt *xprt; 2672ffc2e518SChuck Lever struct sock_xprt *new; 2673c8541ecdSChuck Lever 267496802a09SFrank van Maarseveen if (args->addrlen > sizeof(xprt->addr)) { 2675c8541ecdSChuck Lever dprintk("RPC: xs_setup_xprt: address too large\n"); 2676c8541ecdSChuck Lever return ERR_PTR(-EBADF); 2677c8541ecdSChuck Lever } 2678c8541ecdSChuck Lever 2679d9ba131dSTrond Myklebust xprt = xprt_alloc(args->net, sizeof(*new), slot_table_size, 2680d9ba131dSTrond Myklebust max_slot_table_size); 2681bd1722d4SPavel Emelyanov if (xprt == NULL) { 268246121cf7SChuck Lever dprintk("RPC: xs_setup_xprt: couldn't allocate " 268346121cf7SChuck Lever "rpc_xprt\n"); 2684c8541ecdSChuck Lever return ERR_PTR(-ENOMEM); 2685c8541ecdSChuck Lever } 2686c8541ecdSChuck Lever 2687bd1722d4SPavel Emelyanov new = container_of(xprt, struct sock_xprt, xprt); 2688*edc1b01cSTrond Myklebust mutex_init(&new->recv_mutex); 268996802a09SFrank van Maarseveen memcpy(&xprt->addr, args->dstaddr, args->addrlen); 269096802a09SFrank van Maarseveen xprt->addrlen = args->addrlen; 2691d3bc9a1dSFrank van Maarseveen if (args->srcaddr) 2692fbfffbd5SChuck Lever memcpy(&new->srcaddr, args->srcaddr, args->addrlen); 269392476850SChuck Lever else { 269492476850SChuck Lever int err; 269592476850SChuck Lever err = xs_init_anyaddr(args->dstaddr->sa_family, 269692476850SChuck Lever (struct sockaddr *)&new->srcaddr); 26972aa13531SStanislav Kinsbursky if (err != 0) { 26982aa13531SStanislav Kinsbursky xprt_free(xprt); 269992476850SChuck Lever return ERR_PTR(err); 270092476850SChuck Lever } 27012aa13531SStanislav Kinsbursky } 2702c8541ecdSChuck Lever 2703c8541ecdSChuck Lever return xprt; 2704c8541ecdSChuck Lever } 2705c8541ecdSChuck Lever 2706176e21eeSChuck Lever static const struct rpc_timeout xs_local_default_timeout = { 2707176e21eeSChuck Lever .to_initval = 10 * HZ, 2708176e21eeSChuck Lever .to_maxval = 10 * HZ, 2709176e21eeSChuck Lever .to_retries = 2, 2710176e21eeSChuck Lever }; 2711176e21eeSChuck Lever 2712176e21eeSChuck Lever /** 2713176e21eeSChuck Lever * xs_setup_local - Set up transport to use an AF_LOCAL socket 2714176e21eeSChuck Lever * @args: rpc transport creation arguments 2715176e21eeSChuck Lever * 2716176e21eeSChuck Lever * AF_LOCAL is a "tpi_cots_ord" transport, just like TCP 2717176e21eeSChuck Lever */ 2718176e21eeSChuck Lever static struct rpc_xprt *xs_setup_local(struct xprt_create *args) 2719176e21eeSChuck Lever { 2720176e21eeSChuck Lever struct sockaddr_un *sun = (struct sockaddr_un *)args->dstaddr; 2721176e21eeSChuck Lever struct sock_xprt *transport; 2722176e21eeSChuck Lever struct rpc_xprt *xprt; 2723176e21eeSChuck Lever struct rpc_xprt *ret; 2724176e21eeSChuck Lever 2725d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries, 2726d9ba131dSTrond Myklebust xprt_max_tcp_slot_table_entries); 2727176e21eeSChuck Lever if (IS_ERR(xprt)) 2728176e21eeSChuck Lever return xprt; 2729176e21eeSChuck Lever transport = container_of(xprt, struct sock_xprt, xprt); 2730176e21eeSChuck Lever 2731176e21eeSChuck Lever xprt->prot = 0; 2732176e21eeSChuck Lever xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32); 2733176e21eeSChuck Lever xprt->max_payload = RPC_MAX_FRAGMENT_SIZE; 2734176e21eeSChuck Lever 2735176e21eeSChuck Lever xprt->bind_timeout = XS_BIND_TO; 2736176e21eeSChuck Lever xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 2737176e21eeSChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO; 2738176e21eeSChuck Lever 2739176e21eeSChuck Lever xprt->ops = &xs_local_ops; 2740176e21eeSChuck Lever xprt->timeout = &xs_local_default_timeout; 2741176e21eeSChuck Lever 2742*edc1b01cSTrond Myklebust INIT_WORK(&transport->recv_worker, xs_dummy_data_receive_workfn); 274393dc41bdSNeilBrown INIT_DELAYED_WORK(&transport->connect_worker, 274493dc41bdSNeilBrown xs_dummy_setup_socket); 274593dc41bdSNeilBrown 2746176e21eeSChuck Lever switch (sun->sun_family) { 2747176e21eeSChuck Lever case AF_LOCAL: 2748176e21eeSChuck Lever if (sun->sun_path[0] != '/') { 2749176e21eeSChuck Lever dprintk("RPC: bad AF_LOCAL address: %s\n", 2750176e21eeSChuck Lever sun->sun_path); 2751176e21eeSChuck Lever ret = ERR_PTR(-EINVAL); 2752176e21eeSChuck Lever goto out_err; 2753176e21eeSChuck Lever } 2754176e21eeSChuck Lever xprt_set_bound(xprt); 2755176e21eeSChuck Lever xs_format_peer_addresses(xprt, "local", RPCBIND_NETID_LOCAL); 27567073ea87SJ. Bruce Fields ret = ERR_PTR(xs_local_setup_socket(transport)); 27577073ea87SJ. Bruce Fields if (ret) 27587073ea87SJ. Bruce Fields goto out_err; 2759176e21eeSChuck Lever break; 2760176e21eeSChuck Lever default: 2761176e21eeSChuck Lever ret = ERR_PTR(-EAFNOSUPPORT); 2762176e21eeSChuck Lever goto out_err; 2763176e21eeSChuck Lever } 2764176e21eeSChuck Lever 2765176e21eeSChuck Lever dprintk("RPC: set up xprt to %s via AF_LOCAL\n", 2766176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR]); 2767176e21eeSChuck Lever 2768176e21eeSChuck Lever if (try_module_get(THIS_MODULE)) 2769176e21eeSChuck Lever return xprt; 2770176e21eeSChuck Lever ret = ERR_PTR(-EINVAL); 2771176e21eeSChuck Lever out_err: 2772315f3812SKinglong Mee xs_xprt_free(xprt); 2773176e21eeSChuck Lever return ret; 2774176e21eeSChuck Lever } 2775176e21eeSChuck Lever 27762881ae74STrond Myklebust static const struct rpc_timeout xs_udp_default_timeout = { 27772881ae74STrond Myklebust .to_initval = 5 * HZ, 27782881ae74STrond Myklebust .to_maxval = 30 * HZ, 27792881ae74STrond Myklebust .to_increment = 5 * HZ, 27802881ae74STrond Myklebust .to_retries = 5, 27812881ae74STrond Myklebust }; 27822881ae74STrond Myklebust 27839903cd1cSChuck Lever /** 27849903cd1cSChuck Lever * xs_setup_udp - Set up transport to use a UDP socket 278596802a09SFrank van Maarseveen * @args: rpc transport creation arguments 27869903cd1cSChuck Lever * 27879903cd1cSChuck Lever */ 2788483066d6SAdrian Bunk static struct rpc_xprt *xs_setup_udp(struct xprt_create *args) 2789a246b010SChuck Lever { 27908f9d5b1aSChuck Lever struct sockaddr *addr = args->dstaddr; 2791c8541ecdSChuck Lever struct rpc_xprt *xprt; 2792c8475461SChuck Lever struct sock_xprt *transport; 27930a68b0beSJ. Bruce Fields struct rpc_xprt *ret; 2794a246b010SChuck Lever 2795d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_udp_slot_table_entries, 2796d9ba131dSTrond Myklebust xprt_udp_slot_table_entries); 2797c8541ecdSChuck Lever if (IS_ERR(xprt)) 2798c8541ecdSChuck Lever return xprt; 2799c8475461SChuck Lever transport = container_of(xprt, struct sock_xprt, xprt); 2800a246b010SChuck Lever 2801ec739ef0SChuck Lever xprt->prot = IPPROTO_UDP; 2802808012fbSChuck Lever xprt->tsh_size = 0; 2803a246b010SChuck Lever /* XXX: header size can vary due to auth type, IPv6, etc. */ 2804a246b010SChuck Lever xprt->max_payload = (1U << 16) - (MAX_HEADER << 3); 2805a246b010SChuck Lever 280603bf4b70SChuck Lever xprt->bind_timeout = XS_BIND_TO; 280703bf4b70SChuck Lever xprt->reestablish_timeout = XS_UDP_REEST_TO; 280803bf4b70SChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO; 2809a246b010SChuck Lever 2810262965f5SChuck Lever xprt->ops = &xs_udp_ops; 2811a246b010SChuck Lever 2812ba7392bbSTrond Myklebust xprt->timeout = &xs_udp_default_timeout; 2813a246b010SChuck Lever 2814*edc1b01cSTrond Myklebust INIT_WORK(&transport->recv_worker, xs_dummy_data_receive_workfn); 2815*edc1b01cSTrond Myklebust INIT_DELAYED_WORK(&transport->connect_worker, xs_udp_setup_socket); 2816*edc1b01cSTrond Myklebust 28178f9d5b1aSChuck Lever switch (addr->sa_family) { 28188f9d5b1aSChuck Lever case AF_INET: 28198f9d5b1aSChuck Lever if (((struct sockaddr_in *)addr)->sin_port != htons(0)) 28208f9d5b1aSChuck Lever xprt_set_bound(xprt); 28218f9d5b1aSChuck Lever 28229dc3b095SChuck Lever xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP); 28238f9d5b1aSChuck Lever break; 28248f9d5b1aSChuck Lever case AF_INET6: 28258f9d5b1aSChuck Lever if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0)) 28268f9d5b1aSChuck Lever xprt_set_bound(xprt); 28278f9d5b1aSChuck Lever 28289dc3b095SChuck Lever xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP6); 28298f9d5b1aSChuck Lever break; 28308f9d5b1aSChuck Lever default: 28310a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT); 28320a68b0beSJ. Bruce Fields goto out_err; 28338f9d5b1aSChuck Lever } 28348f9d5b1aSChuck Lever 2835c740eff8SChuck Lever if (xprt_bound(xprt)) 2836c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (port %s) via %s\n", 2837c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2838c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT], 2839c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 2840c740eff8SChuck Lever else 2841c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (autobind) via %s\n", 2842c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2843c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 2844edb267a6SChuck Lever 2845bc25571eS\"Talpey, Thomas\ if (try_module_get(THIS_MODULE)) 2846c8541ecdSChuck Lever return xprt; 28470a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL); 28480a68b0beSJ. Bruce Fields out_err: 2849315f3812SKinglong Mee xs_xprt_free(xprt); 28500a68b0beSJ. Bruce Fields return ret; 2851a246b010SChuck Lever } 2852a246b010SChuck Lever 28532881ae74STrond Myklebust static const struct rpc_timeout xs_tcp_default_timeout = { 28542881ae74STrond Myklebust .to_initval = 60 * HZ, 28552881ae74STrond Myklebust .to_maxval = 60 * HZ, 28562881ae74STrond Myklebust .to_retries = 2, 28572881ae74STrond Myklebust }; 28582881ae74STrond Myklebust 28599903cd1cSChuck Lever /** 28609903cd1cSChuck Lever * xs_setup_tcp - Set up transport to use a TCP socket 286196802a09SFrank van Maarseveen * @args: rpc transport creation arguments 28629903cd1cSChuck Lever * 28639903cd1cSChuck Lever */ 2864483066d6SAdrian Bunk static struct rpc_xprt *xs_setup_tcp(struct xprt_create *args) 2865a246b010SChuck Lever { 28668f9d5b1aSChuck Lever struct sockaddr *addr = args->dstaddr; 2867c8541ecdSChuck Lever struct rpc_xprt *xprt; 2868c8475461SChuck Lever struct sock_xprt *transport; 28690a68b0beSJ. Bruce Fields struct rpc_xprt *ret; 2870b7993cebSTrond Myklebust unsigned int max_slot_table_size = xprt_max_tcp_slot_table_entries; 2871b7993cebSTrond Myklebust 2872b7993cebSTrond Myklebust if (args->flags & XPRT_CREATE_INFINITE_SLOTS) 2873b7993cebSTrond Myklebust max_slot_table_size = RPC_MAX_SLOT_TABLE_LIMIT; 2874a246b010SChuck Lever 2875d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries, 2876b7993cebSTrond Myklebust max_slot_table_size); 2877c8541ecdSChuck Lever if (IS_ERR(xprt)) 2878c8541ecdSChuck Lever return xprt; 2879c8475461SChuck Lever transport = container_of(xprt, struct sock_xprt, xprt); 2880a246b010SChuck Lever 2881ec739ef0SChuck Lever xprt->prot = IPPROTO_TCP; 2882808012fbSChuck Lever xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32); 2883808012fbSChuck Lever xprt->max_payload = RPC_MAX_FRAGMENT_SIZE; 2884a246b010SChuck Lever 288503bf4b70SChuck Lever xprt->bind_timeout = XS_BIND_TO; 288603bf4b70SChuck Lever xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 288703bf4b70SChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO; 2888a246b010SChuck Lever 2889262965f5SChuck Lever xprt->ops = &xs_tcp_ops; 2890ba7392bbSTrond Myklebust xprt->timeout = &xs_tcp_default_timeout; 2891a246b010SChuck Lever 2892*edc1b01cSTrond Myklebust INIT_WORK(&transport->recv_worker, xs_tcp_data_receive_workfn); 2893*edc1b01cSTrond Myklebust INIT_DELAYED_WORK(&transport->connect_worker, xs_tcp_setup_socket); 2894*edc1b01cSTrond Myklebust 28958f9d5b1aSChuck Lever switch (addr->sa_family) { 28968f9d5b1aSChuck Lever case AF_INET: 28978f9d5b1aSChuck Lever if (((struct sockaddr_in *)addr)->sin_port != htons(0)) 28988f9d5b1aSChuck Lever xprt_set_bound(xprt); 28998f9d5b1aSChuck Lever 29009dc3b095SChuck Lever xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP); 29018f9d5b1aSChuck Lever break; 29028f9d5b1aSChuck Lever case AF_INET6: 29038f9d5b1aSChuck Lever if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0)) 29048f9d5b1aSChuck Lever xprt_set_bound(xprt); 29058f9d5b1aSChuck Lever 29069dc3b095SChuck Lever xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP6); 29078f9d5b1aSChuck Lever break; 29088f9d5b1aSChuck Lever default: 29090a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT); 29100a68b0beSJ. Bruce Fields goto out_err; 29118f9d5b1aSChuck Lever } 29128f9d5b1aSChuck Lever 2913c740eff8SChuck Lever if (xprt_bound(xprt)) 2914c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (port %s) via %s\n", 2915c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2916c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT], 2917c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 2918c740eff8SChuck Lever else 2919c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (autobind) via %s\n", 2920c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2921c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 2922c740eff8SChuck Lever 2923bc25571eS\"Talpey, Thomas\ if (try_module_get(THIS_MODULE)) 2924c8541ecdSChuck Lever return xprt; 29250a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL); 29260a68b0beSJ. Bruce Fields out_err: 2927315f3812SKinglong Mee xs_xprt_free(xprt); 29280a68b0beSJ. Bruce Fields return ret; 2929a246b010SChuck Lever } 2930282b32e1SChuck Lever 2931f300babaSAlexandros Batsakis /** 2932f300babaSAlexandros Batsakis * xs_setup_bc_tcp - Set up transport to use a TCP backchannel socket 2933f300babaSAlexandros Batsakis * @args: rpc transport creation arguments 2934f300babaSAlexandros Batsakis * 2935f300babaSAlexandros Batsakis */ 2936f300babaSAlexandros Batsakis static struct rpc_xprt *xs_setup_bc_tcp(struct xprt_create *args) 2937f300babaSAlexandros Batsakis { 2938f300babaSAlexandros Batsakis struct sockaddr *addr = args->dstaddr; 2939f300babaSAlexandros Batsakis struct rpc_xprt *xprt; 2940f300babaSAlexandros Batsakis struct sock_xprt *transport; 2941f300babaSAlexandros Batsakis struct svc_sock *bc_sock; 29420a68b0beSJ. Bruce Fields struct rpc_xprt *ret; 2943f300babaSAlexandros Batsakis 2944d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries, 2945d9ba131dSTrond Myklebust xprt_tcp_slot_table_entries); 2946f300babaSAlexandros Batsakis if (IS_ERR(xprt)) 2947f300babaSAlexandros Batsakis return xprt; 2948f300babaSAlexandros Batsakis transport = container_of(xprt, struct sock_xprt, xprt); 2949f300babaSAlexandros Batsakis 2950f300babaSAlexandros Batsakis xprt->prot = IPPROTO_TCP; 2951f300babaSAlexandros Batsakis xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32); 2952f300babaSAlexandros Batsakis xprt->max_payload = RPC_MAX_FRAGMENT_SIZE; 2953f300babaSAlexandros Batsakis xprt->timeout = &xs_tcp_default_timeout; 2954f300babaSAlexandros Batsakis 2955f300babaSAlexandros Batsakis /* backchannel */ 2956f300babaSAlexandros Batsakis xprt_set_bound(xprt); 2957f300babaSAlexandros Batsakis xprt->bind_timeout = 0; 2958f300babaSAlexandros Batsakis xprt->reestablish_timeout = 0; 2959f300babaSAlexandros Batsakis xprt->idle_timeout = 0; 2960f300babaSAlexandros Batsakis 2961f300babaSAlexandros Batsakis xprt->ops = &bc_tcp_ops; 2962f300babaSAlexandros Batsakis 2963f300babaSAlexandros Batsakis switch (addr->sa_family) { 2964f300babaSAlexandros Batsakis case AF_INET: 2965f300babaSAlexandros Batsakis xs_format_peer_addresses(xprt, "tcp", 2966f300babaSAlexandros Batsakis RPCBIND_NETID_TCP); 2967f300babaSAlexandros Batsakis break; 2968f300babaSAlexandros Batsakis case AF_INET6: 2969f300babaSAlexandros Batsakis xs_format_peer_addresses(xprt, "tcp", 2970f300babaSAlexandros Batsakis RPCBIND_NETID_TCP6); 2971f300babaSAlexandros Batsakis break; 2972f300babaSAlexandros Batsakis default: 29730a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT); 29740a68b0beSJ. Bruce Fields goto out_err; 2975f300babaSAlexandros Batsakis } 2976f300babaSAlexandros Batsakis 2977f300babaSAlexandros Batsakis dprintk("RPC: set up xprt to %s (port %s) via %s\n", 2978f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_ADDR], 2979f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_PORT], 2980f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_PROTO]); 2981f300babaSAlexandros Batsakis 2982f300babaSAlexandros Batsakis /* 298399de8ea9SJ. Bruce Fields * Once we've associated a backchannel xprt with a connection, 298428303ca3SWeng Meiling * we want to keep it around as long as the connection lasts, 298528303ca3SWeng Meiling * in case we need to start using it for a backchannel again; 298628303ca3SWeng Meiling * this reference won't be dropped until bc_xprt is destroyed. 298799de8ea9SJ. Bruce Fields */ 298899de8ea9SJ. Bruce Fields xprt_get(xprt); 298999de8ea9SJ. Bruce Fields args->bc_xprt->xpt_bc_xprt = xprt; 299099de8ea9SJ. Bruce Fields xprt->bc_xprt = args->bc_xprt; 299199de8ea9SJ. Bruce Fields bc_sock = container_of(args->bc_xprt, struct svc_sock, sk_xprt); 299299de8ea9SJ. Bruce Fields transport->sock = bc_sock->sk_sock; 299399de8ea9SJ. Bruce Fields transport->inet = bc_sock->sk_sk; 299499de8ea9SJ. Bruce Fields 299599de8ea9SJ. Bruce Fields /* 2996f300babaSAlexandros Batsakis * Since we don't want connections for the backchannel, we set 2997f300babaSAlexandros Batsakis * the xprt status to connected 2998f300babaSAlexandros Batsakis */ 2999f300babaSAlexandros Batsakis xprt_set_connected(xprt); 3000f300babaSAlexandros Batsakis 3001f300babaSAlexandros Batsakis if (try_module_get(THIS_MODULE)) 3002f300babaSAlexandros Batsakis return xprt; 3003642aab58SKinglong Mee 3004642aab58SKinglong Mee args->bc_xprt->xpt_bc_xprt = NULL; 300599de8ea9SJ. Bruce Fields xprt_put(xprt); 30060a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL); 30070a68b0beSJ. Bruce Fields out_err: 3008315f3812SKinglong Mee xs_xprt_free(xprt); 30090a68b0beSJ. Bruce Fields return ret; 3010f300babaSAlexandros Batsakis } 3011f300babaSAlexandros Batsakis 3012176e21eeSChuck Lever static struct xprt_class xs_local_transport = { 3013176e21eeSChuck Lever .list = LIST_HEAD_INIT(xs_local_transport.list), 3014176e21eeSChuck Lever .name = "named UNIX socket", 3015176e21eeSChuck Lever .owner = THIS_MODULE, 3016176e21eeSChuck Lever .ident = XPRT_TRANSPORT_LOCAL, 3017176e21eeSChuck Lever .setup = xs_setup_local, 3018176e21eeSChuck Lever }; 3019176e21eeSChuck Lever 3020bc25571eS\"Talpey, Thomas\ static struct xprt_class xs_udp_transport = { 3021bc25571eS\"Talpey, Thomas\ .list = LIST_HEAD_INIT(xs_udp_transport.list), 3022bc25571eS\"Talpey, Thomas\ .name = "udp", 3023bc25571eS\"Talpey, Thomas\ .owner = THIS_MODULE, 3024f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_UDP, 3025bc25571eS\"Talpey, Thomas\ .setup = xs_setup_udp, 3026bc25571eS\"Talpey, Thomas\ }; 3027bc25571eS\"Talpey, Thomas\ 3028bc25571eS\"Talpey, Thomas\ static struct xprt_class xs_tcp_transport = { 3029bc25571eS\"Talpey, Thomas\ .list = LIST_HEAD_INIT(xs_tcp_transport.list), 3030bc25571eS\"Talpey, Thomas\ .name = "tcp", 3031bc25571eS\"Talpey, Thomas\ .owner = THIS_MODULE, 3032f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_TCP, 3033bc25571eS\"Talpey, Thomas\ .setup = xs_setup_tcp, 3034bc25571eS\"Talpey, Thomas\ }; 3035bc25571eS\"Talpey, Thomas\ 3036f300babaSAlexandros Batsakis static struct xprt_class xs_bc_tcp_transport = { 3037f300babaSAlexandros Batsakis .list = LIST_HEAD_INIT(xs_bc_tcp_transport.list), 3038f300babaSAlexandros Batsakis .name = "tcp NFSv4.1 backchannel", 3039f300babaSAlexandros Batsakis .owner = THIS_MODULE, 3040f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_BC_TCP, 3041f300babaSAlexandros Batsakis .setup = xs_setup_bc_tcp, 3042f300babaSAlexandros Batsakis }; 3043f300babaSAlexandros Batsakis 3044282b32e1SChuck Lever /** 3045bc25571eS\"Talpey, Thomas\ * init_socket_xprt - set up xprtsock's sysctls, register with RPC client 3046282b32e1SChuck Lever * 3047282b32e1SChuck Lever */ 3048282b32e1SChuck Lever int init_socket_xprt(void) 3049282b32e1SChuck Lever { 3050f895b252SJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) 30512b1bec5fSEric W. Biederman if (!sunrpc_table_header) 30520b4d4147SEric W. Biederman sunrpc_table_header = register_sysctl_table(sunrpc_table); 3053fbf76683SChuck Lever #endif 3054fbf76683SChuck Lever 3055176e21eeSChuck Lever xprt_register_transport(&xs_local_transport); 3056bc25571eS\"Talpey, Thomas\ xprt_register_transport(&xs_udp_transport); 3057bc25571eS\"Talpey, Thomas\ xprt_register_transport(&xs_tcp_transport); 3058f300babaSAlexandros Batsakis xprt_register_transport(&xs_bc_tcp_transport); 3059bc25571eS\"Talpey, Thomas\ 3060282b32e1SChuck Lever return 0; 3061282b32e1SChuck Lever } 3062282b32e1SChuck Lever 3063282b32e1SChuck Lever /** 3064bc25571eS\"Talpey, Thomas\ * cleanup_socket_xprt - remove xprtsock's sysctls, unregister 3065282b32e1SChuck Lever * 3066282b32e1SChuck Lever */ 3067282b32e1SChuck Lever void cleanup_socket_xprt(void) 3068282b32e1SChuck Lever { 3069f895b252SJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) 3070fbf76683SChuck Lever if (sunrpc_table_header) { 3071fbf76683SChuck Lever unregister_sysctl_table(sunrpc_table_header); 3072fbf76683SChuck Lever sunrpc_table_header = NULL; 3073fbf76683SChuck Lever } 3074fbf76683SChuck Lever #endif 3075bc25571eS\"Talpey, Thomas\ 3076176e21eeSChuck Lever xprt_unregister_transport(&xs_local_transport); 3077bc25571eS\"Talpey, Thomas\ xprt_unregister_transport(&xs_udp_transport); 3078bc25571eS\"Talpey, Thomas\ xprt_unregister_transport(&xs_tcp_transport); 3079f300babaSAlexandros Batsakis xprt_unregister_transport(&xs_bc_tcp_transport); 3080282b32e1SChuck Lever } 3081cbf11071STrond Myklebust 30829bbb9e5aSRusty Russell static int param_set_uint_minmax(const char *val, 30839bbb9e5aSRusty Russell const struct kernel_param *kp, 3084cbf11071STrond Myklebust unsigned int min, unsigned int max) 3085cbf11071STrond Myklebust { 308600cfaa94SDaniel Walter unsigned int num; 3087cbf11071STrond Myklebust int ret; 3088cbf11071STrond Myklebust 3089cbf11071STrond Myklebust if (!val) 3090cbf11071STrond Myklebust return -EINVAL; 309100cfaa94SDaniel Walter ret = kstrtouint(val, 0, &num); 3092cbf11071STrond Myklebust if (ret == -EINVAL || num < min || num > max) 3093cbf11071STrond Myklebust return -EINVAL; 3094cbf11071STrond Myklebust *((unsigned int *)kp->arg) = num; 3095cbf11071STrond Myklebust return 0; 3096cbf11071STrond Myklebust } 3097cbf11071STrond Myklebust 30989bbb9e5aSRusty Russell static int param_set_portnr(const char *val, const struct kernel_param *kp) 3099cbf11071STrond Myklebust { 3100cbf11071STrond Myklebust return param_set_uint_minmax(val, kp, 3101cbf11071STrond Myklebust RPC_MIN_RESVPORT, 3102cbf11071STrond Myklebust RPC_MAX_RESVPORT); 3103cbf11071STrond Myklebust } 3104cbf11071STrond Myklebust 31059c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_portnr = { 31069bbb9e5aSRusty Russell .set = param_set_portnr, 31079bbb9e5aSRusty Russell .get = param_get_uint, 31089bbb9e5aSRusty Russell }; 31099bbb9e5aSRusty Russell 3110cbf11071STrond Myklebust #define param_check_portnr(name, p) \ 3111cbf11071STrond Myklebust __param_check(name, p, unsigned int); 3112cbf11071STrond Myklebust 3113cbf11071STrond Myklebust module_param_named(min_resvport, xprt_min_resvport, portnr, 0644); 3114cbf11071STrond Myklebust module_param_named(max_resvport, xprt_max_resvport, portnr, 0644); 3115cbf11071STrond Myklebust 31169bbb9e5aSRusty Russell static int param_set_slot_table_size(const char *val, 31179bbb9e5aSRusty Russell const struct kernel_param *kp) 3118cbf11071STrond Myklebust { 3119cbf11071STrond Myklebust return param_set_uint_minmax(val, kp, 3120cbf11071STrond Myklebust RPC_MIN_SLOT_TABLE, 3121cbf11071STrond Myklebust RPC_MAX_SLOT_TABLE); 3122cbf11071STrond Myklebust } 3123cbf11071STrond Myklebust 31249c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_slot_table_size = { 31259bbb9e5aSRusty Russell .set = param_set_slot_table_size, 31269bbb9e5aSRusty Russell .get = param_get_uint, 31279bbb9e5aSRusty Russell }; 31289bbb9e5aSRusty Russell 3129cbf11071STrond Myklebust #define param_check_slot_table_size(name, p) \ 3130cbf11071STrond Myklebust __param_check(name, p, unsigned int); 3131cbf11071STrond Myklebust 3132d9ba131dSTrond Myklebust static int param_set_max_slot_table_size(const char *val, 3133d9ba131dSTrond Myklebust const struct kernel_param *kp) 3134d9ba131dSTrond Myklebust { 3135d9ba131dSTrond Myklebust return param_set_uint_minmax(val, kp, 3136d9ba131dSTrond Myklebust RPC_MIN_SLOT_TABLE, 3137d9ba131dSTrond Myklebust RPC_MAX_SLOT_TABLE_LIMIT); 3138d9ba131dSTrond Myklebust } 3139d9ba131dSTrond Myklebust 31409c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_max_slot_table_size = { 3141d9ba131dSTrond Myklebust .set = param_set_max_slot_table_size, 3142d9ba131dSTrond Myklebust .get = param_get_uint, 3143d9ba131dSTrond Myklebust }; 3144d9ba131dSTrond Myklebust 3145d9ba131dSTrond Myklebust #define param_check_max_slot_table_size(name, p) \ 3146d9ba131dSTrond Myklebust __param_check(name, p, unsigned int); 3147d9ba131dSTrond Myklebust 3148cbf11071STrond Myklebust module_param_named(tcp_slot_table_entries, xprt_tcp_slot_table_entries, 3149cbf11071STrond Myklebust slot_table_size, 0644); 3150d9ba131dSTrond Myklebust module_param_named(tcp_max_slot_table_entries, xprt_max_tcp_slot_table_entries, 3151d9ba131dSTrond Myklebust max_slot_table_size, 0644); 3152cbf11071STrond Myklebust module_param_named(udp_slot_table_entries, xprt_udp_slot_table_entries, 3153cbf11071STrond Myklebust slot_table_size, 0644); 3154cbf11071STrond Myklebust 3155