1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 2a246b010SChuck Lever /* 3a246b010SChuck Lever * linux/net/sunrpc/xprtsock.c 4a246b010SChuck Lever * 5a246b010SChuck Lever * Client-side transport implementation for sockets. 6a246b010SChuck Lever * 7113aa838SAlan Cox * TCP callback races fixes (C) 1998 Red Hat 8113aa838SAlan Cox * TCP send fixes (C) 1998 Red Hat 9a246b010SChuck Lever * TCP NFS related read + write fixes 10a246b010SChuck Lever * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie> 11a246b010SChuck Lever * 12a246b010SChuck Lever * Rewrite of larges part of the code in order to stabilize TCP stuff. 13a246b010SChuck Lever * Fix behaviour when socket buffer is full. 14a246b010SChuck Lever * (C) 1999 Trond Myklebust <trond.myklebust@fys.uio.no> 1555aa4f58SChuck Lever * 1655aa4f58SChuck Lever * IP socket transport implementation, (C) 2005 Chuck Lever <cel@netapp.com> 178f9d5b1aSChuck Lever * 188f9d5b1aSChuck Lever * IPv6 support contributed by Gilles Quillard, Bull Open Source, 2005. 198f9d5b1aSChuck Lever * <gilles.quillard@bull.net> 20a246b010SChuck Lever */ 21a246b010SChuck Lever 22a246b010SChuck Lever #include <linux/types.h> 23176e21eeSChuck Lever #include <linux/string.h> 24a246b010SChuck Lever #include <linux/slab.h> 25bc25571eS\"Talpey, Thomas\ #include <linux/module.h> 26a246b010SChuck Lever #include <linux/capability.h> 27a246b010SChuck Lever #include <linux/pagemap.h> 28a246b010SChuck Lever #include <linux/errno.h> 29a246b010SChuck Lever #include <linux/socket.h> 30a246b010SChuck Lever #include <linux/in.h> 31a246b010SChuck Lever #include <linux/net.h> 32a246b010SChuck Lever #include <linux/mm.h> 33176e21eeSChuck Lever #include <linux/un.h> 34a246b010SChuck Lever #include <linux/udp.h> 35a246b010SChuck Lever #include <linux/tcp.h> 36a246b010SChuck Lever #include <linux/sunrpc/clnt.h> 375976687aSJeff Layton #include <linux/sunrpc/addr.h> 3802107148SChuck Lever #include <linux/sunrpc/sched.h> 394cfc7e60SRahul Iyer #include <linux/sunrpc/svcsock.h> 4049c36fccS\"Talpey, Thomas\ #include <linux/sunrpc/xprtsock.h> 41a246b010SChuck Lever #include <linux/file.h> 429e00abc3STrond Myklebust #ifdef CONFIG_SUNRPC_BACKCHANNEL 4344b98efdSRicardo Labiaga #include <linux/sunrpc/bc_xprt.h> 4444b98efdSRicardo Labiaga #endif 45a246b010SChuck Lever 46a246b010SChuck Lever #include <net/sock.h> 47a246b010SChuck Lever #include <net/checksum.h> 48a246b010SChuck Lever #include <net/udp.h> 49a246b010SChuck Lever #include <net/tcp.h> 50277e4ab7STrond Myklebust #include <linux/bvec.h> 516a829eb8STrond Myklebust #include <linux/highmem.h> 52277e4ab7STrond Myklebust #include <linux/uio.h> 53*a1231fdaSTrond Myklebust #include <linux/sched/mm.h> 54a246b010SChuck Lever 5540b5ea0cSTrond Myklebust #include <trace/events/sunrpc.h> 5640b5ea0cSTrond Myklebust 574cfc7e60SRahul Iyer #include "sunrpc.h" 58176e21eeSChuck Lever 59176e21eeSChuck Lever static void xs_close(struct rpc_xprt *xprt); 607196dbb0STrond Myklebust static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt, 617196dbb0STrond Myklebust struct socket *sock); 62176e21eeSChuck Lever 639903cd1cSChuck Lever /* 64c556b754SChuck Lever * xprtsock tunables 65c556b754SChuck Lever */ 6609acfea5STrond Myklebust static unsigned int xprt_udp_slot_table_entries = RPC_DEF_SLOT_TABLE; 6709acfea5STrond Myklebust static unsigned int xprt_tcp_slot_table_entries = RPC_MIN_SLOT_TABLE; 6809acfea5STrond Myklebust static unsigned int xprt_max_tcp_slot_table_entries = RPC_MAX_SLOT_TABLE; 69c556b754SChuck Lever 7009acfea5STrond Myklebust static unsigned int xprt_min_resvport = RPC_DEF_MIN_RESVPORT; 7109acfea5STrond Myklebust static unsigned int xprt_max_resvport = RPC_DEF_MAX_RESVPORT; 72c556b754SChuck Lever 737d1e8255STrond Myklebust #define XS_TCP_LINGER_TO (15U * HZ) 7425fe6142STrond Myklebust static unsigned int xs_tcp_fin_timeout __read_mostly = XS_TCP_LINGER_TO; 757d1e8255STrond Myklebust 76c556b754SChuck Lever /* 77fbf76683SChuck Lever * We can register our own files under /proc/sys/sunrpc by 78fbf76683SChuck Lever * calling register_sysctl_table() again. The files in that 79fbf76683SChuck Lever * directory become the union of all files registered there. 80fbf76683SChuck Lever * 81fbf76683SChuck Lever * We simply need to make sure that we don't collide with 82fbf76683SChuck Lever * someone else's file names! 83fbf76683SChuck Lever */ 84fbf76683SChuck Lever 85fbf76683SChuck Lever static unsigned int min_slot_table_size = RPC_MIN_SLOT_TABLE; 86fbf76683SChuck Lever static unsigned int max_slot_table_size = RPC_MAX_SLOT_TABLE; 87d9ba131dSTrond Myklebust static unsigned int max_tcp_slot_table_limit = RPC_MAX_SLOT_TABLE_LIMIT; 88fbf76683SChuck Lever static unsigned int xprt_min_resvport_limit = RPC_MIN_RESVPORT; 89fbf76683SChuck Lever static unsigned int xprt_max_resvport_limit = RPC_MAX_RESVPORT; 90fbf76683SChuck Lever 91fbf76683SChuck Lever static struct ctl_table_header *sunrpc_table_header; 92fbf76683SChuck Lever 93fbf76683SChuck Lever /* 94fbf76683SChuck Lever * FIXME: changing the UDP slot table size should also resize the UDP 95fbf76683SChuck Lever * socket buffers for existing UDP transports 96fbf76683SChuck Lever */ 97fe2c6338SJoe Perches static struct ctl_table xs_tunables_table[] = { 98fbf76683SChuck Lever { 99fbf76683SChuck Lever .procname = "udp_slot_table_entries", 100fbf76683SChuck Lever .data = &xprt_udp_slot_table_entries, 101fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 102fbf76683SChuck Lever .mode = 0644, 1036d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 104fbf76683SChuck Lever .extra1 = &min_slot_table_size, 105fbf76683SChuck Lever .extra2 = &max_slot_table_size 106fbf76683SChuck Lever }, 107fbf76683SChuck Lever { 108fbf76683SChuck Lever .procname = "tcp_slot_table_entries", 109fbf76683SChuck Lever .data = &xprt_tcp_slot_table_entries, 110fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 111fbf76683SChuck Lever .mode = 0644, 1126d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 113fbf76683SChuck Lever .extra1 = &min_slot_table_size, 114fbf76683SChuck Lever .extra2 = &max_slot_table_size 115fbf76683SChuck Lever }, 116fbf76683SChuck Lever { 117d9ba131dSTrond Myklebust .procname = "tcp_max_slot_table_entries", 118d9ba131dSTrond Myklebust .data = &xprt_max_tcp_slot_table_entries, 119d9ba131dSTrond Myklebust .maxlen = sizeof(unsigned int), 120d9ba131dSTrond Myklebust .mode = 0644, 121d9ba131dSTrond Myklebust .proc_handler = proc_dointvec_minmax, 122d9ba131dSTrond Myklebust .extra1 = &min_slot_table_size, 123d9ba131dSTrond Myklebust .extra2 = &max_tcp_slot_table_limit 124d9ba131dSTrond Myklebust }, 125d9ba131dSTrond Myklebust { 126fbf76683SChuck Lever .procname = "min_resvport", 127fbf76683SChuck Lever .data = &xprt_min_resvport, 128fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 129fbf76683SChuck Lever .mode = 0644, 1306d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 131fbf76683SChuck Lever .extra1 = &xprt_min_resvport_limit, 132826799e6SJ. Bruce Fields .extra2 = &xprt_max_resvport_limit 133fbf76683SChuck Lever }, 134fbf76683SChuck Lever { 135fbf76683SChuck Lever .procname = "max_resvport", 136fbf76683SChuck Lever .data = &xprt_max_resvport, 137fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 138fbf76683SChuck Lever .mode = 0644, 1396d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 140826799e6SJ. Bruce Fields .extra1 = &xprt_min_resvport_limit, 141fbf76683SChuck Lever .extra2 = &xprt_max_resvport_limit 142fbf76683SChuck Lever }, 143fbf76683SChuck Lever { 14425fe6142STrond Myklebust .procname = "tcp_fin_timeout", 14525fe6142STrond Myklebust .data = &xs_tcp_fin_timeout, 14625fe6142STrond Myklebust .maxlen = sizeof(xs_tcp_fin_timeout), 14725fe6142STrond Myklebust .mode = 0644, 1486d456111SEric W. Biederman .proc_handler = proc_dointvec_jiffies, 14925fe6142STrond Myklebust }, 150f8572d8fSEric W. Biederman { }, 151fbf76683SChuck Lever }; 152fbf76683SChuck Lever 153fe2c6338SJoe Perches static struct ctl_table sunrpc_table[] = { 154fbf76683SChuck Lever { 155fbf76683SChuck Lever .procname = "sunrpc", 156fbf76683SChuck Lever .mode = 0555, 157fbf76683SChuck Lever .child = xs_tunables_table 158fbf76683SChuck Lever }, 159f8572d8fSEric W. Biederman { }, 160fbf76683SChuck Lever }; 161fbf76683SChuck Lever 162fbf76683SChuck Lever /* 16303bf4b70SChuck Lever * Wait duration for a reply from the RPC portmapper. 16403bf4b70SChuck Lever */ 16503bf4b70SChuck Lever #define XS_BIND_TO (60U * HZ) 16603bf4b70SChuck Lever 16703bf4b70SChuck Lever /* 16803bf4b70SChuck Lever * Delay if a UDP socket connect error occurs. This is most likely some 16903bf4b70SChuck Lever * kind of resource problem on the local host. 17003bf4b70SChuck Lever */ 17103bf4b70SChuck Lever #define XS_UDP_REEST_TO (2U * HZ) 17203bf4b70SChuck Lever 17303bf4b70SChuck Lever /* 17403bf4b70SChuck Lever * The reestablish timeout allows clients to delay for a bit before attempting 17503bf4b70SChuck Lever * to reconnect to a server that just dropped our connection. 17603bf4b70SChuck Lever * 17703bf4b70SChuck Lever * We implement an exponential backoff when trying to reestablish a TCP 17803bf4b70SChuck Lever * transport connection with the server. Some servers like to drop a TCP 17903bf4b70SChuck Lever * connection when they are overworked, so we start with a short timeout and 18003bf4b70SChuck Lever * increase over time if the server is down or not responding. 18103bf4b70SChuck Lever */ 18203bf4b70SChuck Lever #define XS_TCP_INIT_REEST_TO (3U * HZ) 18303bf4b70SChuck Lever 18403bf4b70SChuck Lever /* 18503bf4b70SChuck Lever * TCP idle timeout; client drops the transport socket if it is idle 18603bf4b70SChuck Lever * for this long. Note that we also timeout UDP sockets to prevent 18703bf4b70SChuck Lever * holding port numbers when there is no RPC traffic. 18803bf4b70SChuck Lever */ 18903bf4b70SChuck Lever #define XS_IDLE_DISC_TO (5U * 60 * HZ) 19003bf4b70SChuck Lever 191f895b252SJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) 192a246b010SChuck Lever # undef RPC_DEBUG_DATA 1939903cd1cSChuck Lever # define RPCDBG_FACILITY RPCDBG_TRANS 194a246b010SChuck Lever #endif 195a246b010SChuck Lever 196a246b010SChuck Lever #ifdef RPC_DEBUG_DATA 1979903cd1cSChuck Lever static void xs_pktdump(char *msg, u32 *packet, unsigned int count) 198a246b010SChuck Lever { 199a246b010SChuck Lever u8 *buf = (u8 *) packet; 200a246b010SChuck Lever int j; 201a246b010SChuck Lever 202a246b010SChuck Lever dprintk("RPC: %s\n", msg); 203a246b010SChuck Lever for (j = 0; j < count && j < 128; j += 4) { 204a246b010SChuck Lever if (!(j & 31)) { 205a246b010SChuck Lever if (j) 206a246b010SChuck Lever dprintk("\n"); 207a246b010SChuck Lever dprintk("0x%04x ", j); 208a246b010SChuck Lever } 209a246b010SChuck Lever dprintk("%02x%02x%02x%02x ", 210a246b010SChuck Lever buf[j], buf[j+1], buf[j+2], buf[j+3]); 211a246b010SChuck Lever } 212a246b010SChuck Lever dprintk("\n"); 213a246b010SChuck Lever } 214a246b010SChuck Lever #else 2159903cd1cSChuck Lever static inline void xs_pktdump(char *msg, u32 *packet, unsigned int count) 216a246b010SChuck Lever { 217a246b010SChuck Lever /* NOP */ 218a246b010SChuck Lever } 219a246b010SChuck Lever #endif 220a246b010SChuck Lever 2212118071dSTrond Myklebust static inline struct rpc_xprt *xprt_from_sock(struct sock *sk) 2222118071dSTrond Myklebust { 2232118071dSTrond Myklebust return (struct rpc_xprt *) sk->sk_user_data; 2242118071dSTrond Myklebust } 2252118071dSTrond Myklebust 22695392c59SChuck Lever static inline struct sockaddr *xs_addr(struct rpc_xprt *xprt) 227edb267a6SChuck Lever { 22895392c59SChuck Lever return (struct sockaddr *) &xprt->addr; 22995392c59SChuck Lever } 23095392c59SChuck Lever 231176e21eeSChuck Lever static inline struct sockaddr_un *xs_addr_un(struct rpc_xprt *xprt) 232176e21eeSChuck Lever { 233176e21eeSChuck Lever return (struct sockaddr_un *) &xprt->addr; 234176e21eeSChuck Lever } 235176e21eeSChuck Lever 23695392c59SChuck Lever static inline struct sockaddr_in *xs_addr_in(struct rpc_xprt *xprt) 23795392c59SChuck Lever { 23895392c59SChuck Lever return (struct sockaddr_in *) &xprt->addr; 23995392c59SChuck Lever } 24095392c59SChuck Lever 24195392c59SChuck Lever static inline struct sockaddr_in6 *xs_addr_in6(struct rpc_xprt *xprt) 24295392c59SChuck Lever { 24395392c59SChuck Lever return (struct sockaddr_in6 *) &xprt->addr; 24495392c59SChuck Lever } 24595392c59SChuck Lever 246c877b849SChuck Lever static void xs_format_common_peer_addresses(struct rpc_xprt *xprt) 247c877b849SChuck Lever { 248c877b849SChuck Lever struct sockaddr *sap = xs_addr(xprt); 2499dc3b095SChuck Lever struct sockaddr_in6 *sin6; 2509dc3b095SChuck Lever struct sockaddr_in *sin; 251176e21eeSChuck Lever struct sockaddr_un *sun; 252c877b849SChuck Lever char buf[128]; 253c877b849SChuck Lever 2549dc3b095SChuck Lever switch (sap->sa_family) { 255176e21eeSChuck Lever case AF_LOCAL: 256176e21eeSChuck Lever sun = xs_addr_un(xprt); 257176e21eeSChuck Lever strlcpy(buf, sun->sun_path, sizeof(buf)); 258176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] = 259176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL); 260176e21eeSChuck Lever break; 2619dc3b095SChuck Lever case AF_INET: 262176e21eeSChuck Lever (void)rpc_ntop(sap, buf, sizeof(buf)); 263176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] = 264176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL); 2659dc3b095SChuck Lever sin = xs_addr_in(xprt); 266fc0b5791SJoe Perches snprintf(buf, sizeof(buf), "%08x", ntohl(sin->sin_addr.s_addr)); 2679dc3b095SChuck Lever break; 2689dc3b095SChuck Lever case AF_INET6: 269176e21eeSChuck Lever (void)rpc_ntop(sap, buf, sizeof(buf)); 270176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] = 271176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL); 2729dc3b095SChuck Lever sin6 = xs_addr_in6(xprt); 273fc0b5791SJoe Perches snprintf(buf, sizeof(buf), "%pi6", &sin6->sin6_addr); 2749dc3b095SChuck Lever break; 2759dc3b095SChuck Lever default: 2769dc3b095SChuck Lever BUG(); 2779dc3b095SChuck Lever } 278176e21eeSChuck Lever 2799dc3b095SChuck Lever xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL); 2809dc3b095SChuck Lever } 2819dc3b095SChuck Lever 2829dc3b095SChuck Lever static void xs_format_common_peer_ports(struct rpc_xprt *xprt) 2839dc3b095SChuck Lever { 2849dc3b095SChuck Lever struct sockaddr *sap = xs_addr(xprt); 2859dc3b095SChuck Lever char buf[128]; 2869dc3b095SChuck Lever 28781160e66SJoe Perches snprintf(buf, sizeof(buf), "%u", rpc_get_port(sap)); 288c877b849SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL); 289c877b849SChuck Lever 29081160e66SJoe Perches snprintf(buf, sizeof(buf), "%4hx", rpc_get_port(sap)); 291c877b849SChuck Lever xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL); 292c877b849SChuck Lever } 293c877b849SChuck Lever 2949dc3b095SChuck Lever static void xs_format_peer_addresses(struct rpc_xprt *xprt, 295b454ae90SChuck Lever const char *protocol, 296b454ae90SChuck Lever const char *netid) 297edb267a6SChuck Lever { 298b454ae90SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO] = protocol; 299b454ae90SChuck Lever xprt->address_strings[RPC_DISPLAY_NETID] = netid; 300c877b849SChuck Lever xs_format_common_peer_addresses(xprt); 3019dc3b095SChuck Lever xs_format_common_peer_ports(xprt); 302edb267a6SChuck Lever } 303edb267a6SChuck Lever 3049dc3b095SChuck Lever static void xs_update_peer_port(struct rpc_xprt *xprt) 3054b6473fbSChuck Lever { 3069dc3b095SChuck Lever kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]); 3079dc3b095SChuck Lever kfree(xprt->address_strings[RPC_DISPLAY_PORT]); 3084b6473fbSChuck Lever 3099dc3b095SChuck Lever xs_format_common_peer_ports(xprt); 310edb267a6SChuck Lever } 311edb267a6SChuck Lever 312edb267a6SChuck Lever static void xs_free_peer_addresses(struct rpc_xprt *xprt) 313edb267a6SChuck Lever { 31433e01dc7SChuck Lever unsigned int i; 31533e01dc7SChuck Lever 31633e01dc7SChuck Lever for (i = 0; i < RPC_DISPLAY_MAX; i++) 31733e01dc7SChuck Lever switch (i) { 31833e01dc7SChuck Lever case RPC_DISPLAY_PROTO: 31933e01dc7SChuck Lever case RPC_DISPLAY_NETID: 32033e01dc7SChuck Lever continue; 32133e01dc7SChuck Lever default: 32233e01dc7SChuck Lever kfree(xprt->address_strings[i]); 32333e01dc7SChuck Lever } 324edb267a6SChuck Lever } 325edb267a6SChuck Lever 326277e4ab7STrond Myklebust static size_t 327277e4ab7STrond Myklebust xs_alloc_sparse_pages(struct xdr_buf *buf, size_t want, gfp_t gfp) 328277e4ab7STrond Myklebust { 329277e4ab7STrond Myklebust size_t i,n; 330277e4ab7STrond Myklebust 33116e5e90fSTrond Myklebust if (!want || !(buf->flags & XDRBUF_SPARSE_PAGES)) 332277e4ab7STrond Myklebust return want; 333277e4ab7STrond Myklebust n = (buf->page_base + want + PAGE_SIZE - 1) >> PAGE_SHIFT; 334277e4ab7STrond Myklebust for (i = 0; i < n; i++) { 335277e4ab7STrond Myklebust if (buf->pages[i]) 336277e4ab7STrond Myklebust continue; 337277e4ab7STrond Myklebust buf->bvec[i].bv_page = buf->pages[i] = alloc_page(gfp); 338277e4ab7STrond Myklebust if (!buf->pages[i]) { 33916e5e90fSTrond Myklebust i *= PAGE_SIZE; 34016e5e90fSTrond Myklebust return i > buf->page_base ? i - buf->page_base : 0; 341277e4ab7STrond Myklebust } 342277e4ab7STrond Myklebust } 343277e4ab7STrond Myklebust return want; 344277e4ab7STrond Myklebust } 345277e4ab7STrond Myklebust 346277e4ab7STrond Myklebust static ssize_t 347277e4ab7STrond Myklebust xs_sock_recvmsg(struct socket *sock, struct msghdr *msg, int flags, size_t seek) 348277e4ab7STrond Myklebust { 349277e4ab7STrond Myklebust ssize_t ret; 350277e4ab7STrond Myklebust if (seek != 0) 351277e4ab7STrond Myklebust iov_iter_advance(&msg->msg_iter, seek); 352277e4ab7STrond Myklebust ret = sock_recvmsg(sock, msg, flags); 353277e4ab7STrond Myklebust return ret > 0 ? ret + seek : ret; 354277e4ab7STrond Myklebust } 355277e4ab7STrond Myklebust 356277e4ab7STrond Myklebust static ssize_t 357277e4ab7STrond Myklebust xs_read_kvec(struct socket *sock, struct msghdr *msg, int flags, 358277e4ab7STrond Myklebust struct kvec *kvec, size_t count, size_t seek) 359277e4ab7STrond Myklebust { 3600e9b4a82SAl Viro iov_iter_kvec(&msg->msg_iter, READ, kvec, 1, count); 361277e4ab7STrond Myklebust return xs_sock_recvmsg(sock, msg, flags, seek); 362277e4ab7STrond Myklebust } 363277e4ab7STrond Myklebust 364277e4ab7STrond Myklebust static ssize_t 365277e4ab7STrond Myklebust xs_read_bvec(struct socket *sock, struct msghdr *msg, int flags, 366277e4ab7STrond Myklebust struct bio_vec *bvec, unsigned long nr, size_t count, 367277e4ab7STrond Myklebust size_t seek) 368277e4ab7STrond Myklebust { 3690e9b4a82SAl Viro iov_iter_bvec(&msg->msg_iter, READ, bvec, nr, count); 370277e4ab7STrond Myklebust return xs_sock_recvmsg(sock, msg, flags, seek); 371277e4ab7STrond Myklebust } 372277e4ab7STrond Myklebust 373277e4ab7STrond Myklebust static ssize_t 374277e4ab7STrond Myklebust xs_read_discard(struct socket *sock, struct msghdr *msg, int flags, 375277e4ab7STrond Myklebust size_t count) 376277e4ab7STrond Myklebust { 377b76a5afdSTrond Myklebust iov_iter_discard(&msg->msg_iter, READ, count); 378b76a5afdSTrond Myklebust return sock_recvmsg(sock, msg, flags); 379277e4ab7STrond Myklebust } 380277e4ab7STrond Myklebust 3816a829eb8STrond Myklebust #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 3826a829eb8STrond Myklebust static void 3836a829eb8STrond Myklebust xs_flush_bvec(const struct bio_vec *bvec, size_t count, size_t seek) 3846a829eb8STrond Myklebust { 3856a829eb8STrond Myklebust struct bvec_iter bi = { 3866a829eb8STrond Myklebust .bi_size = count, 3876a829eb8STrond Myklebust }; 3886a829eb8STrond Myklebust struct bio_vec bv; 3896a829eb8STrond Myklebust 3906a829eb8STrond Myklebust bvec_iter_advance(bvec, &bi, seek & PAGE_MASK); 3916a829eb8STrond Myklebust for_each_bvec(bv, bvec, bi, bi) 3926a829eb8STrond Myklebust flush_dcache_page(bv.bv_page); 3936a829eb8STrond Myklebust } 3946a829eb8STrond Myklebust #else 3956a829eb8STrond Myklebust static inline void 3966a829eb8STrond Myklebust xs_flush_bvec(const struct bio_vec *bvec, size_t count, size_t seek) 3976a829eb8STrond Myklebust { 3986a829eb8STrond Myklebust } 3996a829eb8STrond Myklebust #endif 4006a829eb8STrond Myklebust 401277e4ab7STrond Myklebust static ssize_t 402277e4ab7STrond Myklebust xs_read_xdr_buf(struct socket *sock, struct msghdr *msg, int flags, 403277e4ab7STrond Myklebust struct xdr_buf *buf, size_t count, size_t seek, size_t *read) 404277e4ab7STrond Myklebust { 405277e4ab7STrond Myklebust size_t want, seek_init = seek, offset = 0; 406277e4ab7STrond Myklebust ssize_t ret; 407277e4ab7STrond Myklebust 408277e4ab7STrond Myklebust if (seek < buf->head[0].iov_len) { 409277e4ab7STrond Myklebust want = min_t(size_t, count, buf->head[0].iov_len); 410277e4ab7STrond Myklebust ret = xs_read_kvec(sock, msg, flags, &buf->head[0], want, seek); 411277e4ab7STrond Myklebust if (ret <= 0) 412277e4ab7STrond Myklebust goto sock_err; 413277e4ab7STrond Myklebust offset += ret; 414277e4ab7STrond Myklebust if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC)) 415277e4ab7STrond Myklebust goto out; 416277e4ab7STrond Myklebust if (ret != want) 417c4433055STrond Myklebust goto out; 418277e4ab7STrond Myklebust seek = 0; 419277e4ab7STrond Myklebust } else { 420277e4ab7STrond Myklebust seek -= buf->head[0].iov_len; 421277e4ab7STrond Myklebust offset += buf->head[0].iov_len; 422277e4ab7STrond Myklebust } 42316e5e90fSTrond Myklebust 424277e4ab7STrond Myklebust want = xs_alloc_sparse_pages(buf, 425277e4ab7STrond Myklebust min_t(size_t, count - offset, buf->page_len), 426277e4ab7STrond Myklebust GFP_NOWAIT); 42716e5e90fSTrond Myklebust if (seek < want) { 428277e4ab7STrond Myklebust ret = xs_read_bvec(sock, msg, flags, buf->bvec, 429277e4ab7STrond Myklebust xdr_buf_pagecount(buf), 430277e4ab7STrond Myklebust want + buf->page_base, 431277e4ab7STrond Myklebust seek + buf->page_base); 432277e4ab7STrond Myklebust if (ret <= 0) 433277e4ab7STrond Myklebust goto sock_err; 4346a829eb8STrond Myklebust xs_flush_bvec(buf->bvec, ret, seek + buf->page_base); 435277e4ab7STrond Myklebust offset += ret - buf->page_base; 436277e4ab7STrond Myklebust if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC)) 437277e4ab7STrond Myklebust goto out; 438277e4ab7STrond Myklebust if (ret != want) 439c4433055STrond Myklebust goto out; 440277e4ab7STrond Myklebust seek = 0; 441277e4ab7STrond Myklebust } else { 44216e5e90fSTrond Myklebust seek -= want; 44316e5e90fSTrond Myklebust offset += want; 444277e4ab7STrond Myklebust } 44516e5e90fSTrond Myklebust 446277e4ab7STrond Myklebust if (seek < buf->tail[0].iov_len) { 447277e4ab7STrond Myklebust want = min_t(size_t, count - offset, buf->tail[0].iov_len); 448277e4ab7STrond Myklebust ret = xs_read_kvec(sock, msg, flags, &buf->tail[0], want, seek); 449277e4ab7STrond Myklebust if (ret <= 0) 450277e4ab7STrond Myklebust goto sock_err; 451277e4ab7STrond Myklebust offset += ret; 452277e4ab7STrond Myklebust if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC)) 453277e4ab7STrond Myklebust goto out; 454277e4ab7STrond Myklebust if (ret != want) 455c4433055STrond Myklebust goto out; 456277e4ab7STrond Myklebust } else 457277e4ab7STrond Myklebust offset += buf->tail[0].iov_len; 458277e4ab7STrond Myklebust ret = -EMSGSIZE; 459277e4ab7STrond Myklebust out: 460277e4ab7STrond Myklebust *read = offset - seek_init; 461277e4ab7STrond Myklebust return ret; 462277e4ab7STrond Myklebust sock_err: 463277e4ab7STrond Myklebust offset += seek; 464277e4ab7STrond Myklebust goto out; 465277e4ab7STrond Myklebust } 466277e4ab7STrond Myklebust 467277e4ab7STrond Myklebust static void 468277e4ab7STrond Myklebust xs_read_header(struct sock_xprt *transport, struct xdr_buf *buf) 469277e4ab7STrond Myklebust { 470277e4ab7STrond Myklebust if (!transport->recv.copied) { 471277e4ab7STrond Myklebust if (buf->head[0].iov_len >= transport->recv.offset) 472277e4ab7STrond Myklebust memcpy(buf->head[0].iov_base, 473277e4ab7STrond Myklebust &transport->recv.xid, 474277e4ab7STrond Myklebust transport->recv.offset); 475277e4ab7STrond Myklebust transport->recv.copied = transport->recv.offset; 476277e4ab7STrond Myklebust } 477277e4ab7STrond Myklebust } 478277e4ab7STrond Myklebust 479277e4ab7STrond Myklebust static bool 480277e4ab7STrond Myklebust xs_read_stream_request_done(struct sock_xprt *transport) 481277e4ab7STrond Myklebust { 482277e4ab7STrond Myklebust return transport->recv.fraghdr & cpu_to_be32(RPC_LAST_STREAM_FRAGMENT); 483277e4ab7STrond Myklebust } 484277e4ab7STrond Myklebust 485277e4ab7STrond Myklebust static ssize_t 486277e4ab7STrond Myklebust xs_read_stream_request(struct sock_xprt *transport, struct msghdr *msg, 487277e4ab7STrond Myklebust int flags, struct rpc_rqst *req) 488277e4ab7STrond Myklebust { 489277e4ab7STrond Myklebust struct xdr_buf *buf = &req->rq_private_buf; 490277e4ab7STrond Myklebust size_t want, read; 491277e4ab7STrond Myklebust ssize_t ret; 492277e4ab7STrond Myklebust 493277e4ab7STrond Myklebust xs_read_header(transport, buf); 494277e4ab7STrond Myklebust 495277e4ab7STrond Myklebust want = transport->recv.len - transport->recv.offset; 496277e4ab7STrond Myklebust ret = xs_read_xdr_buf(transport->sock, msg, flags, buf, 497277e4ab7STrond Myklebust transport->recv.copied + want, transport->recv.copied, 498277e4ab7STrond Myklebust &read); 499277e4ab7STrond Myklebust transport->recv.offset += read; 500277e4ab7STrond Myklebust transport->recv.copied += read; 501277e4ab7STrond Myklebust if (transport->recv.offset == transport->recv.len) { 502277e4ab7STrond Myklebust if (xs_read_stream_request_done(transport)) 503277e4ab7STrond Myklebust msg->msg_flags |= MSG_EOR; 504c4433055STrond Myklebust return read; 505277e4ab7STrond Myklebust } 506277e4ab7STrond Myklebust 507277e4ab7STrond Myklebust switch (ret) { 508c4433055STrond Myklebust default: 509c4433055STrond Myklebust break; 51026781eabSTrond Myklebust case -EFAULT: 511277e4ab7STrond Myklebust case -EMSGSIZE: 51226781eabSTrond Myklebust msg->msg_flags |= MSG_TRUNC; 513c4433055STrond Myklebust return read; 514277e4ab7STrond Myklebust case 0: 515277e4ab7STrond Myklebust return -ESHUTDOWN; 516277e4ab7STrond Myklebust } 517c4433055STrond Myklebust return ret < 0 ? ret : read; 518277e4ab7STrond Myklebust } 519277e4ab7STrond Myklebust 520277e4ab7STrond Myklebust static size_t 521277e4ab7STrond Myklebust xs_read_stream_headersize(bool isfrag) 522277e4ab7STrond Myklebust { 523277e4ab7STrond Myklebust if (isfrag) 524277e4ab7STrond Myklebust return sizeof(__be32); 525277e4ab7STrond Myklebust return 3 * sizeof(__be32); 526277e4ab7STrond Myklebust } 527277e4ab7STrond Myklebust 528277e4ab7STrond Myklebust static ssize_t 529277e4ab7STrond Myklebust xs_read_stream_header(struct sock_xprt *transport, struct msghdr *msg, 530277e4ab7STrond Myklebust int flags, size_t want, size_t seek) 531277e4ab7STrond Myklebust { 532277e4ab7STrond Myklebust struct kvec kvec = { 533277e4ab7STrond Myklebust .iov_base = &transport->recv.fraghdr, 534277e4ab7STrond Myklebust .iov_len = want, 535277e4ab7STrond Myklebust }; 536277e4ab7STrond Myklebust return xs_read_kvec(transport->sock, msg, flags, &kvec, want, seek); 537277e4ab7STrond Myklebust } 538277e4ab7STrond Myklebust 539277e4ab7STrond Myklebust #if defined(CONFIG_SUNRPC_BACKCHANNEL) 540277e4ab7STrond Myklebust static ssize_t 541277e4ab7STrond Myklebust xs_read_stream_call(struct sock_xprt *transport, struct msghdr *msg, int flags) 542277e4ab7STrond Myklebust { 543277e4ab7STrond Myklebust struct rpc_xprt *xprt = &transport->xprt; 544277e4ab7STrond Myklebust struct rpc_rqst *req; 545277e4ab7STrond Myklebust ssize_t ret; 546277e4ab7STrond Myklebust 547277e4ab7STrond Myklebust /* Look up and lock the request corresponding to the given XID */ 548277e4ab7STrond Myklebust req = xprt_lookup_bc_request(xprt, transport->recv.xid); 549277e4ab7STrond Myklebust if (!req) { 550277e4ab7STrond Myklebust printk(KERN_WARNING "Callback slot table overflowed\n"); 551277e4ab7STrond Myklebust return -ESHUTDOWN; 552277e4ab7STrond Myklebust } 553277e4ab7STrond Myklebust 554277e4ab7STrond Myklebust ret = xs_read_stream_request(transport, msg, flags, req); 555277e4ab7STrond Myklebust if (msg->msg_flags & (MSG_EOR|MSG_TRUNC)) 556c4433055STrond Myklebust xprt_complete_bc_request(req, transport->recv.copied); 557277e4ab7STrond Myklebust 558277e4ab7STrond Myklebust return ret; 559277e4ab7STrond Myklebust } 560277e4ab7STrond Myklebust #else /* CONFIG_SUNRPC_BACKCHANNEL */ 561277e4ab7STrond Myklebust static ssize_t 562277e4ab7STrond Myklebust xs_read_stream_call(struct sock_xprt *transport, struct msghdr *msg, int flags) 563277e4ab7STrond Myklebust { 564277e4ab7STrond Myklebust return -ESHUTDOWN; 565277e4ab7STrond Myklebust } 566277e4ab7STrond Myklebust #endif /* CONFIG_SUNRPC_BACKCHANNEL */ 567277e4ab7STrond Myklebust 568277e4ab7STrond Myklebust static ssize_t 569277e4ab7STrond Myklebust xs_read_stream_reply(struct sock_xprt *transport, struct msghdr *msg, int flags) 570277e4ab7STrond Myklebust { 571277e4ab7STrond Myklebust struct rpc_xprt *xprt = &transport->xprt; 572277e4ab7STrond Myklebust struct rpc_rqst *req; 573277e4ab7STrond Myklebust ssize_t ret = 0; 574277e4ab7STrond Myklebust 575277e4ab7STrond Myklebust /* Look up and lock the request corresponding to the given XID */ 576277e4ab7STrond Myklebust spin_lock(&xprt->queue_lock); 577277e4ab7STrond Myklebust req = xprt_lookup_rqst(xprt, transport->recv.xid); 578277e4ab7STrond Myklebust if (!req) { 579277e4ab7STrond Myklebust msg->msg_flags |= MSG_TRUNC; 580277e4ab7STrond Myklebust goto out; 581277e4ab7STrond Myklebust } 582277e4ab7STrond Myklebust xprt_pin_rqst(req); 583277e4ab7STrond Myklebust spin_unlock(&xprt->queue_lock); 584277e4ab7STrond Myklebust 585277e4ab7STrond Myklebust ret = xs_read_stream_request(transport, msg, flags, req); 586277e4ab7STrond Myklebust 587277e4ab7STrond Myklebust spin_lock(&xprt->queue_lock); 588277e4ab7STrond Myklebust if (msg->msg_flags & (MSG_EOR|MSG_TRUNC)) 589c4433055STrond Myklebust xprt_complete_rqst(req->rq_task, transport->recv.copied); 590277e4ab7STrond Myklebust xprt_unpin_rqst(req); 591277e4ab7STrond Myklebust out: 592277e4ab7STrond Myklebust spin_unlock(&xprt->queue_lock); 593277e4ab7STrond Myklebust return ret; 594277e4ab7STrond Myklebust } 595277e4ab7STrond Myklebust 596277e4ab7STrond Myklebust static ssize_t 597277e4ab7STrond Myklebust xs_read_stream(struct sock_xprt *transport, int flags) 598277e4ab7STrond Myklebust { 599277e4ab7STrond Myklebust struct msghdr msg = { 0 }; 600277e4ab7STrond Myklebust size_t want, read = 0; 601277e4ab7STrond Myklebust ssize_t ret = 0; 602277e4ab7STrond Myklebust 603277e4ab7STrond Myklebust if (transport->recv.len == 0) { 604277e4ab7STrond Myklebust want = xs_read_stream_headersize(transport->recv.copied != 0); 605277e4ab7STrond Myklebust ret = xs_read_stream_header(transport, &msg, flags, want, 606277e4ab7STrond Myklebust transport->recv.offset); 607277e4ab7STrond Myklebust if (ret <= 0) 608277e4ab7STrond Myklebust goto out_err; 609277e4ab7STrond Myklebust transport->recv.offset = ret; 610c4433055STrond Myklebust if (transport->recv.offset != want) 611c4433055STrond Myklebust return transport->recv.offset; 612277e4ab7STrond Myklebust transport->recv.len = be32_to_cpu(transport->recv.fraghdr) & 613277e4ab7STrond Myklebust RPC_FRAGMENT_SIZE_MASK; 614277e4ab7STrond Myklebust transport->recv.offset -= sizeof(transport->recv.fraghdr); 615277e4ab7STrond Myklebust read = ret; 616277e4ab7STrond Myklebust } 617277e4ab7STrond Myklebust 618277e4ab7STrond Myklebust switch (be32_to_cpu(transport->recv.calldir)) { 619c4433055STrond Myklebust default: 620c4433055STrond Myklebust msg.msg_flags |= MSG_TRUNC; 621c4433055STrond Myklebust break; 622277e4ab7STrond Myklebust case RPC_CALL: 623277e4ab7STrond Myklebust ret = xs_read_stream_call(transport, &msg, flags); 624277e4ab7STrond Myklebust break; 625277e4ab7STrond Myklebust case RPC_REPLY: 626277e4ab7STrond Myklebust ret = xs_read_stream_reply(transport, &msg, flags); 627277e4ab7STrond Myklebust } 628277e4ab7STrond Myklebust if (msg.msg_flags & MSG_TRUNC) { 629277e4ab7STrond Myklebust transport->recv.calldir = cpu_to_be32(-1); 630277e4ab7STrond Myklebust transport->recv.copied = -1; 631277e4ab7STrond Myklebust } 632277e4ab7STrond Myklebust if (ret < 0) 633277e4ab7STrond Myklebust goto out_err; 634277e4ab7STrond Myklebust read += ret; 635277e4ab7STrond Myklebust if (transport->recv.offset < transport->recv.len) { 636c4433055STrond Myklebust if (!(msg.msg_flags & MSG_TRUNC)) 637c4433055STrond Myklebust return read; 638b76a5afdSTrond Myklebust msg.msg_flags = 0; 639277e4ab7STrond Myklebust ret = xs_read_discard(transport->sock, &msg, flags, 640277e4ab7STrond Myklebust transport->recv.len - transport->recv.offset); 641277e4ab7STrond Myklebust if (ret <= 0) 642277e4ab7STrond Myklebust goto out_err; 643277e4ab7STrond Myklebust transport->recv.offset += ret; 644277e4ab7STrond Myklebust read += ret; 645277e4ab7STrond Myklebust if (transport->recv.offset != transport->recv.len) 646c4433055STrond Myklebust return read; 647277e4ab7STrond Myklebust } 648277e4ab7STrond Myklebust if (xs_read_stream_request_done(transport)) { 649c50b8ee0STrond Myklebust trace_xs_stream_read_request(transport); 650277e4ab7STrond Myklebust transport->recv.copied = 0; 651277e4ab7STrond Myklebust } 652277e4ab7STrond Myklebust transport->recv.offset = 0; 653277e4ab7STrond Myklebust transport->recv.len = 0; 654277e4ab7STrond Myklebust return read; 655277e4ab7STrond Myklebust out_err: 65679462857STrond Myklebust return ret != 0 ? ret : -ESHUTDOWN; 657277e4ab7STrond Myklebust } 658277e4ab7STrond Myklebust 659c50b8ee0STrond Myklebust static void xs_stream_data_receive(struct sock_xprt *transport) 660c50b8ee0STrond Myklebust { 661c50b8ee0STrond Myklebust size_t read = 0; 662c50b8ee0STrond Myklebust ssize_t ret = 0; 663c50b8ee0STrond Myklebust 664c50b8ee0STrond Myklebust mutex_lock(&transport->recv_mutex); 665dfcf0380STrond Myklebust clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state); 666c50b8ee0STrond Myklebust if (transport->sock == NULL) 667c50b8ee0STrond Myklebust goto out; 668c50b8ee0STrond Myklebust for (;;) { 669c50b8ee0STrond Myklebust ret = xs_read_stream(transport, MSG_DONTWAIT); 670c4433055STrond Myklebust if (ret < 0) 671c50b8ee0STrond Myklebust break; 672c50b8ee0STrond Myklebust read += ret; 673c50b8ee0STrond Myklebust cond_resched(); 674c50b8ee0STrond Myklebust } 675c50b8ee0STrond Myklebust out: 676c50b8ee0STrond Myklebust mutex_unlock(&transport->recv_mutex); 677c50b8ee0STrond Myklebust trace_xs_stream_read_data(&transport->xprt, ret, read); 678c50b8ee0STrond Myklebust } 679c50b8ee0STrond Myklebust 680c50b8ee0STrond Myklebust static void xs_stream_data_receive_workfn(struct work_struct *work) 681c50b8ee0STrond Myklebust { 682c50b8ee0STrond Myklebust struct sock_xprt *transport = 683c50b8ee0STrond Myklebust container_of(work, struct sock_xprt, recv_worker); 684*a1231fdaSTrond Myklebust unsigned int pflags = memalloc_nofs_save(); 685*a1231fdaSTrond Myklebust 686c50b8ee0STrond Myklebust xs_stream_data_receive(transport); 687*a1231fdaSTrond Myklebust memalloc_nofs_restore(pflags); 688c50b8ee0STrond Myklebust } 689c50b8ee0STrond Myklebust 690550aebfeSTrond Myklebust static void 691550aebfeSTrond Myklebust xs_stream_reset_connect(struct sock_xprt *transport) 692550aebfeSTrond Myklebust { 693550aebfeSTrond Myklebust transport->recv.offset = 0; 694550aebfeSTrond Myklebust transport->recv.len = 0; 695550aebfeSTrond Myklebust transport->recv.copied = 0; 696550aebfeSTrond Myklebust transport->xmit.offset = 0; 697550aebfeSTrond Myklebust transport->xprt.stat.connect_count++; 698550aebfeSTrond Myklebust transport->xprt.stat.connect_start = jiffies; 699550aebfeSTrond Myklebust } 700550aebfeSTrond Myklebust 701b4b5cc85SChuck Lever #define XS_SENDMSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL) 702b4b5cc85SChuck Lever 70324c5684bSTrond Myklebust static int xs_send_kvec(struct socket *sock, struct sockaddr *addr, int addrlen, struct kvec *vec, unsigned int base, int more) 704b4b5cc85SChuck Lever { 705b4b5cc85SChuck Lever struct msghdr msg = { 706b4b5cc85SChuck Lever .msg_name = addr, 707b4b5cc85SChuck Lever .msg_namelen = addrlen, 70824c5684bSTrond Myklebust .msg_flags = XS_SENDMSG_FLAGS | (more ? MSG_MORE : 0), 70924c5684bSTrond Myklebust }; 71024c5684bSTrond Myklebust struct kvec iov = { 71124c5684bSTrond Myklebust .iov_base = vec->iov_base + base, 71224c5684bSTrond Myklebust .iov_len = vec->iov_len - base, 713b4b5cc85SChuck Lever }; 714b4b5cc85SChuck Lever 71524c5684bSTrond Myklebust if (iov.iov_len != 0) 716b4b5cc85SChuck Lever return kernel_sendmsg(sock, &msg, &iov, 1, iov.iov_len); 717b4b5cc85SChuck Lever return kernel_sendmsg(sock, &msg, NULL, 0, 0); 718b4b5cc85SChuck Lever } 719b4b5cc85SChuck Lever 720f279cd00SJason Baron static int xs_send_pagedata(struct socket *sock, struct xdr_buf *xdr, unsigned int base, int more, bool zerocopy, int *sent_p) 721b4b5cc85SChuck Lever { 722a6b31d18STrond Myklebust ssize_t (*do_sendpage)(struct socket *sock, struct page *page, 723a6b31d18STrond Myklebust int offset, size_t size, int flags); 72424c5684bSTrond Myklebust struct page **ppage; 72524c5684bSTrond Myklebust unsigned int remainder; 726f279cd00SJason Baron int err; 727b4b5cc85SChuck Lever 72824c5684bSTrond Myklebust remainder = xdr->page_len - base; 72924c5684bSTrond Myklebust base += xdr->page_base; 73024c5684bSTrond Myklebust ppage = xdr->pages + (base >> PAGE_SHIFT); 73124c5684bSTrond Myklebust base &= ~PAGE_MASK; 732a6b31d18STrond Myklebust do_sendpage = sock->ops->sendpage; 733a6b31d18STrond Myklebust if (!zerocopy) 734a6b31d18STrond Myklebust do_sendpage = sock_no_sendpage; 73524c5684bSTrond Myklebust for(;;) { 73624c5684bSTrond Myklebust unsigned int len = min_t(unsigned int, PAGE_SIZE - base, remainder); 73724c5684bSTrond Myklebust int flags = XS_SENDMSG_FLAGS; 73824c5684bSTrond Myklebust 73924c5684bSTrond Myklebust remainder -= len; 74031303d6cSTrond Myklebust if (more) 74124c5684bSTrond Myklebust flags |= MSG_MORE; 74231303d6cSTrond Myklebust if (remainder != 0) 74331303d6cSTrond Myklebust flags |= MSG_SENDPAGE_NOTLAST | MSG_MORE; 744a6b31d18STrond Myklebust err = do_sendpage(sock, *ppage, base, len, flags); 74524c5684bSTrond Myklebust if (remainder == 0 || err != len) 74624c5684bSTrond Myklebust break; 747f279cd00SJason Baron *sent_p += err; 74824c5684bSTrond Myklebust ppage++; 74924c5684bSTrond Myklebust base = 0; 75024c5684bSTrond Myklebust } 751f279cd00SJason Baron if (err > 0) { 752f279cd00SJason Baron *sent_p += err; 753f279cd00SJason Baron err = 0; 754f279cd00SJason Baron } 75524c5684bSTrond Myklebust return err; 756b4b5cc85SChuck Lever } 757b4b5cc85SChuck Lever 7589903cd1cSChuck Lever /** 7599903cd1cSChuck Lever * xs_sendpages - write pages directly to a socket 7609903cd1cSChuck Lever * @sock: socket to send on 7619903cd1cSChuck Lever * @addr: UDP only -- address of destination 7629903cd1cSChuck Lever * @addrlen: UDP only -- length of destination address 7639903cd1cSChuck Lever * @xdr: buffer containing this request 7649903cd1cSChuck Lever * @base: starting position in the buffer 765a6b31d18STrond Myklebust * @zerocopy: true if it is safe to use sendpage() 766f279cd00SJason Baron * @sent_p: return the total number of bytes successfully queued for sending 7679903cd1cSChuck Lever * 768a246b010SChuck Lever */ 769f279cd00SJason Baron static int xs_sendpages(struct socket *sock, struct sockaddr *addr, int addrlen, struct xdr_buf *xdr, unsigned int base, bool zerocopy, int *sent_p) 770a246b010SChuck Lever { 77124c5684bSTrond Myklebust unsigned int remainder = xdr->len - base; 772f279cd00SJason Baron int err = 0; 773f279cd00SJason Baron int sent = 0; 774a246b010SChuck Lever 775262965f5SChuck Lever if (unlikely(!sock)) 776fba91afbSTrond Myklebust return -ENOTSOCK; 777262965f5SChuck Lever 77824c5684bSTrond Myklebust if (base != 0) { 77924c5684bSTrond Myklebust addr = NULL; 78024c5684bSTrond Myklebust addrlen = 0; 78124c5684bSTrond Myklebust } 782262965f5SChuck Lever 78324c5684bSTrond Myklebust if (base < xdr->head[0].iov_len || addr != NULL) { 78424c5684bSTrond Myklebust unsigned int len = xdr->head[0].iov_len - base; 78524c5684bSTrond Myklebust remainder -= len; 78624c5684bSTrond Myklebust err = xs_send_kvec(sock, addr, addrlen, &xdr->head[0], base, remainder != 0); 78724c5684bSTrond Myklebust if (remainder == 0 || err != len) 788a246b010SChuck Lever goto out; 789f279cd00SJason Baron *sent_p += err; 790a246b010SChuck Lever base = 0; 791a246b010SChuck Lever } else 79224c5684bSTrond Myklebust base -= xdr->head[0].iov_len; 793a246b010SChuck Lever 79424c5684bSTrond Myklebust if (base < xdr->page_len) { 79524c5684bSTrond Myklebust unsigned int len = xdr->page_len - base; 79624c5684bSTrond Myklebust remainder -= len; 797f279cd00SJason Baron err = xs_send_pagedata(sock, xdr, base, remainder != 0, zerocopy, &sent); 798f279cd00SJason Baron *sent_p += sent; 799f279cd00SJason Baron if (remainder == 0 || sent != len) 800a246b010SChuck Lever goto out; 801a246b010SChuck Lever base = 0; 80224c5684bSTrond Myklebust } else 80324c5684bSTrond Myklebust base -= xdr->page_len; 80424c5684bSTrond Myklebust 80524c5684bSTrond Myklebust if (base >= xdr->tail[0].iov_len) 806f279cd00SJason Baron return 0; 80724c5684bSTrond Myklebust err = xs_send_kvec(sock, NULL, 0, &xdr->tail[0], base, 0); 808a246b010SChuck Lever out: 809f279cd00SJason Baron if (err > 0) { 810f279cd00SJason Baron *sent_p += err; 811f279cd00SJason Baron err = 0; 812f279cd00SJason Baron } 81324c5684bSTrond Myklebust return err; 814a246b010SChuck Lever } 815a246b010SChuck Lever 8169903cd1cSChuck Lever /** 817c544577dSTrond Myklebust * xs_nospace - handle transmit was incomplete 81850f484e2STrond Myklebust * @req: pointer to RPC request 8199903cd1cSChuck Lever * 820a246b010SChuck Lever */ 821c544577dSTrond Myklebust static int xs_nospace(struct rpc_rqst *req) 822a246b010SChuck Lever { 823262965f5SChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 824ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 82506ea0bfeSTrond Myklebust struct sock *sk = transport->inet; 82624ca9a84STrond Myklebust int ret = -EAGAIN; 827a246b010SChuck Lever 82846121cf7SChuck Lever dprintk("RPC: %5u xmit incomplete (%u left of %u)\n", 829c544577dSTrond Myklebust req->rq_task->tk_pid, 830c544577dSTrond Myklebust req->rq_slen - transport->xmit.offset, 831262965f5SChuck Lever req->rq_slen); 832a246b010SChuck Lever 833262965f5SChuck Lever /* Protect against races with write_space */ 834262965f5SChuck Lever spin_lock_bh(&xprt->transport_lock); 835a246b010SChuck Lever 836262965f5SChuck Lever /* Don't race with disconnect */ 837b6ddf64fSTrond Myklebust if (xprt_connected(xprt)) { 83813331a55STrond Myklebust /* wait for more buffer space */ 83906ea0bfeSTrond Myklebust sk->sk_write_pending++; 840c544577dSTrond Myklebust xprt_wait_for_buffer_space(xprt); 84113331a55STrond Myklebust } else 8425e3771ceSTrond Myklebust ret = -ENOTCONN; 843a246b010SChuck Lever 844262965f5SChuck Lever spin_unlock_bh(&xprt->transport_lock); 84506ea0bfeSTrond Myklebust 84606ea0bfeSTrond Myklebust /* Race breaker in case memory is freed before above code is called */ 847d48f9ce7SDavid Vrabel if (ret == -EAGAIN) { 848d48f9ce7SDavid Vrabel struct socket_wq *wq; 849d48f9ce7SDavid Vrabel 850d48f9ce7SDavid Vrabel rcu_read_lock(); 851d48f9ce7SDavid Vrabel wq = rcu_dereference(sk->sk_wq); 852d48f9ce7SDavid Vrabel set_bit(SOCKWQ_ASYNC_NOSPACE, &wq->flags); 853d48f9ce7SDavid Vrabel rcu_read_unlock(); 854d48f9ce7SDavid Vrabel 85506ea0bfeSTrond Myklebust sk->sk_write_space(sk); 856d48f9ce7SDavid Vrabel } 8575e3771ceSTrond Myklebust return ret; 858a246b010SChuck Lever } 859a246b010SChuck Lever 860277e4ab7STrond Myklebust static void 861277e4ab7STrond Myklebust xs_stream_prepare_request(struct rpc_rqst *req) 862277e4ab7STrond Myklebust { 863277e4ab7STrond Myklebust req->rq_task->tk_status = xdr_alloc_bvec(&req->rq_rcv_buf, GFP_NOIO); 864277e4ab7STrond Myklebust } 865277e4ab7STrond Myklebust 86661677eeeSChuck Lever /* 8674cd34e7cSTrond Myklebust * Determine if the previous message in the stream was aborted before it 8684cd34e7cSTrond Myklebust * could complete transmission. 8694cd34e7cSTrond Myklebust */ 8704cd34e7cSTrond Myklebust static bool 8714cd34e7cSTrond Myklebust xs_send_request_was_aborted(struct sock_xprt *transport, struct rpc_rqst *req) 8724cd34e7cSTrond Myklebust { 8734cd34e7cSTrond Myklebust return transport->xmit.offset != 0 && req->rq_bytes_sent == 0; 8744cd34e7cSTrond Myklebust } 8754cd34e7cSTrond Myklebust 8764cd34e7cSTrond Myklebust /* 87761677eeeSChuck Lever * Construct a stream transport record marker in @buf. 87861677eeeSChuck Lever */ 87961677eeeSChuck Lever static inline void xs_encode_stream_record_marker(struct xdr_buf *buf) 88061677eeeSChuck Lever { 88161677eeeSChuck Lever u32 reclen = buf->len - sizeof(rpc_fraghdr); 88261677eeeSChuck Lever rpc_fraghdr *base = buf->head[0].iov_base; 88361677eeeSChuck Lever *base = cpu_to_be32(RPC_LAST_STREAM_FRAGMENT | reclen); 88461677eeeSChuck Lever } 88561677eeeSChuck Lever 8869903cd1cSChuck Lever /** 887176e21eeSChuck Lever * xs_local_send_request - write an RPC request to an AF_LOCAL socket 88850f484e2STrond Myklebust * @req: pointer to RPC request 889176e21eeSChuck Lever * 890176e21eeSChuck Lever * Return values: 891176e21eeSChuck Lever * 0: The request has been sent 892176e21eeSChuck Lever * EAGAIN: The socket was blocked, please call again later to 893176e21eeSChuck Lever * complete the request 894176e21eeSChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again 895176e21eeSChuck Lever * other: Some other error occured, the request was not sent 896176e21eeSChuck Lever */ 897adfa7144STrond Myklebust static int xs_local_send_request(struct rpc_rqst *req) 898176e21eeSChuck Lever { 899176e21eeSChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 900176e21eeSChuck Lever struct sock_xprt *transport = 901176e21eeSChuck Lever container_of(xprt, struct sock_xprt, xprt); 902176e21eeSChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf; 903176e21eeSChuck Lever int status; 904f279cd00SJason Baron int sent = 0; 905176e21eeSChuck Lever 9064cd34e7cSTrond Myklebust /* Close the stream if the previous transmission was incomplete */ 9074cd34e7cSTrond Myklebust if (xs_send_request_was_aborted(transport, req)) { 9084cd34e7cSTrond Myklebust xs_close(xprt); 9094cd34e7cSTrond Myklebust return -ENOTCONN; 9104cd34e7cSTrond Myklebust } 9114cd34e7cSTrond Myklebust 912176e21eeSChuck Lever xs_encode_stream_record_marker(&req->rq_snd_buf); 913176e21eeSChuck Lever 914176e21eeSChuck Lever xs_pktdump("packet data:", 915176e21eeSChuck Lever req->rq_svec->iov_base, req->rq_svec->iov_len); 916176e21eeSChuck Lever 91778215759SChuck Lever req->rq_xtime = ktime_get(); 9186c7a64e5STrond Myklebust status = xs_sendpages(transport->sock, NULL, 0, xdr, 9196c7a64e5STrond Myklebust transport->xmit.offset, 920f279cd00SJason Baron true, &sent); 921176e21eeSChuck Lever dprintk("RPC: %s(%u) = %d\n", 9226c7a64e5STrond Myklebust __func__, xdr->len - transport->xmit.offset, status); 923743c69e7SNeilBrown 924743c69e7SNeilBrown if (status == -EAGAIN && sock_writeable(transport->inet)) 925743c69e7SNeilBrown status = -ENOBUFS; 926743c69e7SNeilBrown 927f279cd00SJason Baron if (likely(sent > 0) || status == 0) { 9286c7a64e5STrond Myklebust transport->xmit.offset += sent; 9296c7a64e5STrond Myklebust req->rq_bytes_sent = transport->xmit.offset; 930176e21eeSChuck Lever if (likely(req->rq_bytes_sent >= req->rq_slen)) { 9316c7a64e5STrond Myklebust req->rq_xmit_bytes_sent += transport->xmit.offset; 932176e21eeSChuck Lever req->rq_bytes_sent = 0; 9336c7a64e5STrond Myklebust transport->xmit.offset = 0; 934176e21eeSChuck Lever return 0; 935176e21eeSChuck Lever } 936176e21eeSChuck Lever status = -EAGAIN; 937176e21eeSChuck Lever } 938176e21eeSChuck Lever 939176e21eeSChuck Lever switch (status) { 9403601c4a9STrond Myklebust case -ENOBUFS: 941b5872f0cSTrond Myklebust break; 942176e21eeSChuck Lever case -EAGAIN: 943c544577dSTrond Myklebust status = xs_nospace(req); 944176e21eeSChuck Lever break; 945176e21eeSChuck Lever default: 946176e21eeSChuck Lever dprintk("RPC: sendmsg returned unrecognized error %d\n", 947176e21eeSChuck Lever -status); 948e9d47639SGustavo A. R. Silva /* fall through */ 949176e21eeSChuck Lever case -EPIPE: 950176e21eeSChuck Lever xs_close(xprt); 951176e21eeSChuck Lever status = -ENOTCONN; 952176e21eeSChuck Lever } 953176e21eeSChuck Lever 954176e21eeSChuck Lever return status; 955176e21eeSChuck Lever } 956176e21eeSChuck Lever 957176e21eeSChuck Lever /** 958262965f5SChuck Lever * xs_udp_send_request - write an RPC request to a UDP socket 95950f484e2STrond Myklebust * @req: pointer to RPC request 9609903cd1cSChuck Lever * 9619903cd1cSChuck Lever * Return values: 9629903cd1cSChuck Lever * 0: The request has been sent 9639903cd1cSChuck Lever * EAGAIN: The socket was blocked, please call again later to 9649903cd1cSChuck Lever * complete the request 965262965f5SChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again 96625985edcSLucas De Marchi * other: Some other error occurred, the request was not sent 9679903cd1cSChuck Lever */ 968adfa7144STrond Myklebust static int xs_udp_send_request(struct rpc_rqst *req) 969a246b010SChuck Lever { 970a246b010SChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 971ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 972262965f5SChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf; 973f279cd00SJason Baron int sent = 0; 974262965f5SChuck Lever int status; 975262965f5SChuck Lever 976262965f5SChuck Lever xs_pktdump("packet data:", 977262965f5SChuck Lever req->rq_svec->iov_base, 978262965f5SChuck Lever req->rq_svec->iov_len); 979262965f5SChuck Lever 98001d37c42STrond Myklebust if (!xprt_bound(xprt)) 98101d37c42STrond Myklebust return -ENOTCONN; 98275891f50STrond Myklebust 98375891f50STrond Myklebust if (!xprt_request_get_cong(xprt, req)) 98475891f50STrond Myklebust return -EBADSLT; 98575891f50STrond Myklebust 98678215759SChuck Lever req->rq_xtime = ktime_get(); 987f279cd00SJason Baron status = xs_sendpages(transport->sock, xs_addr(xprt), xprt->addrlen, 9886c7a64e5STrond Myklebust xdr, 0, true, &sent); 989262965f5SChuck Lever 990262965f5SChuck Lever dprintk("RPC: xs_udp_send_request(%u) = %d\n", 9916c7a64e5STrond Myklebust xdr->len, status); 992262965f5SChuck Lever 9933dedbb5cSJason Baron /* firewall is blocking us, don't return -EAGAIN or we end up looping */ 9943dedbb5cSJason Baron if (status == -EPERM) 9953dedbb5cSJason Baron goto process_status; 9963dedbb5cSJason Baron 997743c69e7SNeilBrown if (status == -EAGAIN && sock_writeable(transport->inet)) 998743c69e7SNeilBrown status = -ENOBUFS; 999743c69e7SNeilBrown 1000f279cd00SJason Baron if (sent > 0 || status == 0) { 1001f279cd00SJason Baron req->rq_xmit_bytes_sent += sent; 1002f279cd00SJason Baron if (sent >= req->rq_slen) 1003262965f5SChuck Lever return 0; 1004262965f5SChuck Lever /* Still some bytes left; set up for a retry later. */ 1005262965f5SChuck Lever status = -EAGAIN; 10062199700fSTrond Myklebust } 1007262965f5SChuck Lever 10083dedbb5cSJason Baron process_status: 1009262965f5SChuck Lever switch (status) { 1010fba91afbSTrond Myklebust case -ENOTSOCK: 1011fba91afbSTrond Myklebust status = -ENOTCONN; 1012fba91afbSTrond Myklebust /* Should we call xs_close() here? */ 1013fba91afbSTrond Myklebust break; 1014b6ddf64fSTrond Myklebust case -EAGAIN: 1015c544577dSTrond Myklebust status = xs_nospace(req); 1016b6ddf64fSTrond Myklebust break; 1017262965f5SChuck Lever case -ENETUNREACH: 10183601c4a9STrond Myklebust case -ENOBUFS: 1019262965f5SChuck Lever case -EPIPE: 1020262965f5SChuck Lever case -ECONNREFUSED: 10213dedbb5cSJason Baron case -EPERM: 1022262965f5SChuck Lever /* When the server has died, an ICMP port unreachable message 1023262965f5SChuck Lever * prompts ECONNREFUSED. */ 102413331a55STrond Myklebust break; 102513331a55STrond Myklebust default: 102613331a55STrond Myklebust dprintk("RPC: sendmsg returned unrecognized error %d\n", 102713331a55STrond Myklebust -status); 1028262965f5SChuck Lever } 10295fe46e9dSBian Naimeng 1030262965f5SChuck Lever return status; 1031262965f5SChuck Lever } 1032262965f5SChuck Lever 1033e06799f9STrond Myklebust /** 1034262965f5SChuck Lever * xs_tcp_send_request - write an RPC request to a TCP socket 103550f484e2STrond Myklebust * @req: pointer to RPC request 1036262965f5SChuck Lever * 1037262965f5SChuck Lever * Return values: 1038262965f5SChuck Lever * 0: The request has been sent 1039262965f5SChuck Lever * EAGAIN: The socket was blocked, please call again later to 1040262965f5SChuck Lever * complete the request 1041262965f5SChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again 104225985edcSLucas De Marchi * other: Some other error occurred, the request was not sent 1043262965f5SChuck Lever * 1044262965f5SChuck Lever * XXX: In the case of soft timeouts, should we eventually give up 1045262965f5SChuck Lever * if sendmsg is not able to make progress? 1046262965f5SChuck Lever */ 1047adfa7144STrond Myklebust static int xs_tcp_send_request(struct rpc_rqst *req) 1048262965f5SChuck Lever { 1049262965f5SChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 1050ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1051262965f5SChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf; 1052a6b31d18STrond Myklebust bool zerocopy = true; 10539ffadfbcSTrond Myklebust bool vm_wait = false; 1054b595bb15SChuck Lever int status; 1055f279cd00SJason Baron int sent; 1056a246b010SChuck Lever 10574cd34e7cSTrond Myklebust /* Close the stream if the previous transmission was incomplete */ 10584cd34e7cSTrond Myklebust if (xs_send_request_was_aborted(transport, req)) { 10594cd34e7cSTrond Myklebust if (transport->sock != NULL) 10604cd34e7cSTrond Myklebust kernel_sock_shutdown(transport->sock, SHUT_RDWR); 10614cd34e7cSTrond Myklebust return -ENOTCONN; 10624cd34e7cSTrond Myklebust } 10634cd34e7cSTrond Myklebust 106461677eeeSChuck Lever xs_encode_stream_record_marker(&req->rq_snd_buf); 1065262965f5SChuck Lever 1066262965f5SChuck Lever xs_pktdump("packet data:", 1067262965f5SChuck Lever req->rq_svec->iov_base, 1068262965f5SChuck Lever req->rq_svec->iov_len); 1069a6b31d18STrond Myklebust /* Don't use zero copy if this is a resend. If the RPC call 1070a6b31d18STrond Myklebust * completes while the socket holds a reference to the pages, 1071a6b31d18STrond Myklebust * then we may end up resending corrupted data. 1072a6b31d18STrond Myklebust */ 107350f484e2STrond Myklebust if (req->rq_task->tk_flags & RPC_TASK_SENT) 1074a6b31d18STrond Myklebust zerocopy = false; 1075a246b010SChuck Lever 10767196dbb0STrond Myklebust if (test_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state)) 10777196dbb0STrond Myklebust xs_tcp_set_socket_timeouts(xprt, transport->sock); 10787196dbb0STrond Myklebust 1079a246b010SChuck Lever /* Continue transmitting the packet/record. We must be careful 1080a246b010SChuck Lever * to cope with writespace callbacks arriving _after_ we have 1081262965f5SChuck Lever * called sendmsg(). */ 108278215759SChuck Lever req->rq_xtime = ktime_get(); 1083a246b010SChuck Lever while (1) { 1084f279cd00SJason Baron sent = 0; 1085f279cd00SJason Baron status = xs_sendpages(transport->sock, NULL, 0, xdr, 10866c7a64e5STrond Myklebust transport->xmit.offset, 10876c7a64e5STrond Myklebust zerocopy, &sent); 1088a246b010SChuck Lever 1089262965f5SChuck Lever dprintk("RPC: xs_tcp_send_request(%u) = %d\n", 10906c7a64e5STrond Myklebust xdr->len - transport->xmit.offset, status); 1091262965f5SChuck Lever 1092a246b010SChuck Lever /* If we've sent the entire packet, immediately 1093a246b010SChuck Lever * reset the count of bytes sent. */ 10946c7a64e5STrond Myklebust transport->xmit.offset += sent; 10956c7a64e5STrond Myklebust req->rq_bytes_sent = transport->xmit.offset; 1096262965f5SChuck Lever if (likely(req->rq_bytes_sent >= req->rq_slen)) { 10976c7a64e5STrond Myklebust req->rq_xmit_bytes_sent += transport->xmit.offset; 1098a246b010SChuck Lever req->rq_bytes_sent = 0; 10996c7a64e5STrond Myklebust transport->xmit.offset = 0; 1100a246b010SChuck Lever return 0; 1101a246b010SChuck Lever } 1102262965f5SChuck Lever 11039ffadfbcSTrond Myklebust WARN_ON_ONCE(sent == 0 && status == 0); 11049ffadfbcSTrond Myklebust 11059ffadfbcSTrond Myklebust if (status == -EAGAIN ) { 11069ffadfbcSTrond Myklebust /* 11079ffadfbcSTrond Myklebust * Return EAGAIN if we're sure we're hitting the 11089ffadfbcSTrond Myklebust * socket send buffer limits. 11099ffadfbcSTrond Myklebust */ 11109ffadfbcSTrond Myklebust if (test_bit(SOCK_NOSPACE, &transport->sock->flags)) 11119ffadfbcSTrond Myklebust break; 11129ffadfbcSTrond Myklebust /* 11139ffadfbcSTrond Myklebust * Did we hit a memory allocation failure? 11149ffadfbcSTrond Myklebust */ 11159ffadfbcSTrond Myklebust if (sent == 0) { 11169ffadfbcSTrond Myklebust status = -ENOBUFS; 11179ffadfbcSTrond Myklebust if (vm_wait) 11189ffadfbcSTrond Myklebust break; 11199ffadfbcSTrond Myklebust /* Retry, knowing now that we're below the 11209ffadfbcSTrond Myklebust * socket send buffer limit 11219ffadfbcSTrond Myklebust */ 11229ffadfbcSTrond Myklebust vm_wait = true; 11239ffadfbcSTrond Myklebust } 11249ffadfbcSTrond Myklebust continue; 11259ffadfbcSTrond Myklebust } 1126f580dd04STrond Myklebust if (status < 0) 1127f580dd04STrond Myklebust break; 11289ffadfbcSTrond Myklebust vm_wait = false; 1129a246b010SChuck Lever } 1130a246b010SChuck Lever 1131262965f5SChuck Lever switch (status) { 1132fba91afbSTrond Myklebust case -ENOTSOCK: 1133fba91afbSTrond Myklebust status = -ENOTCONN; 1134fba91afbSTrond Myklebust /* Should we call xs_close() here? */ 1135fba91afbSTrond Myklebust break; 1136262965f5SChuck Lever case -EAGAIN: 1137c544577dSTrond Myklebust status = xs_nospace(req); 1138262965f5SChuck Lever break; 1139262965f5SChuck Lever case -ECONNRESET: 1140262965f5SChuck Lever case -ECONNREFUSED: 1141262965f5SChuck Lever case -ENOTCONN: 11423913c78cSTrond Myklebust case -EADDRINUSE: 1143b5872f0cSTrond Myklebust case -ENOBUFS: 1144b9d2bb2eSTrond Myklebust case -EPIPE: 114513331a55STrond Myklebust break; 114613331a55STrond Myklebust default: 114713331a55STrond Myklebust dprintk("RPC: sendmsg returned unrecognized error %d\n", 114813331a55STrond Myklebust -status); 1149a246b010SChuck Lever } 11505fe46e9dSBian Naimeng 1151a246b010SChuck Lever return status; 1152a246b010SChuck Lever } 1153a246b010SChuck Lever 11542a9e1cfaSTrond Myklebust static void xs_save_old_callbacks(struct sock_xprt *transport, struct sock *sk) 11552a9e1cfaSTrond Myklebust { 11562a9e1cfaSTrond Myklebust transport->old_data_ready = sk->sk_data_ready; 11572a9e1cfaSTrond Myklebust transport->old_state_change = sk->sk_state_change; 11582a9e1cfaSTrond Myklebust transport->old_write_space = sk->sk_write_space; 11592118071dSTrond Myklebust transport->old_error_report = sk->sk_error_report; 11602a9e1cfaSTrond Myklebust } 11612a9e1cfaSTrond Myklebust 11622a9e1cfaSTrond Myklebust static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *sk) 11632a9e1cfaSTrond Myklebust { 11642a9e1cfaSTrond Myklebust sk->sk_data_ready = transport->old_data_ready; 11652a9e1cfaSTrond Myklebust sk->sk_state_change = transport->old_state_change; 11662a9e1cfaSTrond Myklebust sk->sk_write_space = transport->old_write_space; 11672118071dSTrond Myklebust sk->sk_error_report = transport->old_error_report; 11682118071dSTrond Myklebust } 11692118071dSTrond Myklebust 117042d42a5bSTrond Myklebust static void xs_sock_reset_state_flags(struct rpc_xprt *xprt) 117142d42a5bSTrond Myklebust { 117242d42a5bSTrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 117342d42a5bSTrond Myklebust 117442d42a5bSTrond Myklebust clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state); 117542d42a5bSTrond Myklebust } 117642d42a5bSTrond Myklebust 1177b70ae915STrond Myklebust static void xs_sock_reset_connection_flags(struct rpc_xprt *xprt) 1178b70ae915STrond Myklebust { 1179b70ae915STrond Myklebust smp_mb__before_atomic(); 1180b70ae915STrond Myklebust clear_bit(XPRT_CLOSE_WAIT, &xprt->state); 1181b70ae915STrond Myklebust clear_bit(XPRT_CLOSING, &xprt->state); 118242d42a5bSTrond Myklebust xs_sock_reset_state_flags(xprt); 1183b70ae915STrond Myklebust smp_mb__after_atomic(); 1184b70ae915STrond Myklebust } 1185b70ae915STrond Myklebust 11862118071dSTrond Myklebust /** 11872118071dSTrond Myklebust * xs_error_report - callback to handle TCP socket state errors 11882118071dSTrond Myklebust * @sk: socket 11892118071dSTrond Myklebust * 11902118071dSTrond Myklebust * Note: we don't call sock_error() since there may be a rpc_task 11912118071dSTrond Myklebust * using the socket, and so we don't want to clear sk->sk_err. 11922118071dSTrond Myklebust */ 11932118071dSTrond Myklebust static void xs_error_report(struct sock *sk) 11942118071dSTrond Myklebust { 11952118071dSTrond Myklebust struct rpc_xprt *xprt; 11962118071dSTrond Myklebust int err; 11972118071dSTrond Myklebust 11982118071dSTrond Myklebust read_lock_bh(&sk->sk_callback_lock); 11992118071dSTrond Myklebust if (!(xprt = xprt_from_sock(sk))) 12002118071dSTrond Myklebust goto out; 12012118071dSTrond Myklebust 12022118071dSTrond Myklebust err = -sk->sk_err; 12032118071dSTrond Myklebust if (err == 0) 12042118071dSTrond Myklebust goto out; 12052118071dSTrond Myklebust dprintk("RPC: xs_error_report client %p, error=%d...\n", 12062118071dSTrond Myklebust xprt, -err); 1207e8353c76STrond Myklebust trace_rpc_socket_error(xprt, sk->sk_socket, err); 12082118071dSTrond Myklebust xprt_wake_pending_tasks(xprt, err); 12092118071dSTrond Myklebust out: 12102118071dSTrond Myklebust read_unlock_bh(&sk->sk_callback_lock); 12112a9e1cfaSTrond Myklebust } 12122a9e1cfaSTrond Myklebust 1213fe315e76SChuck Lever static void xs_reset_transport(struct sock_xprt *transport) 1214a246b010SChuck Lever { 1215ee0ac0c2SChuck Lever struct socket *sock = transport->sock; 1216ee0ac0c2SChuck Lever struct sock *sk = transport->inet; 12176cc7e908STrond Myklebust struct rpc_xprt *xprt = &transport->xprt; 1218a246b010SChuck Lever 1219fe315e76SChuck Lever if (sk == NULL) 1220fe315e76SChuck Lever return; 12219903cd1cSChuck Lever 1222264d1df3SJeff Layton if (atomic_read(&transport->xprt.swapper)) 1223264d1df3SJeff Layton sk_clear_memalloc(sk); 1224264d1df3SJeff Layton 122509939204STrond Myklebust kernel_sock_shutdown(sock, SHUT_RDWR); 122609939204STrond Myklebust 1227edc1b01cSTrond Myklebust mutex_lock(&transport->recv_mutex); 1228a246b010SChuck Lever write_lock_bh(&sk->sk_callback_lock); 1229ee0ac0c2SChuck Lever transport->inet = NULL; 1230ee0ac0c2SChuck Lever transport->sock = NULL; 1231a246b010SChuck Lever 1232a246b010SChuck Lever sk->sk_user_data = NULL; 12332a9e1cfaSTrond Myklebust 12342a9e1cfaSTrond Myklebust xs_restore_old_callbacks(transport, sk); 12350c78789eSTrond Myklebust xprt_clear_connected(xprt); 1236a246b010SChuck Lever write_unlock_bh(&sk->sk_callback_lock); 12376cc7e908STrond Myklebust xs_sock_reset_connection_flags(xprt); 1238edc1b01cSTrond Myklebust mutex_unlock(&transport->recv_mutex); 1239a246b010SChuck Lever 12406cc7e908STrond Myklebust trace_rpc_socket_close(xprt, sock); 1241a246b010SChuck Lever sock_release(sock); 12420445f92cSTrond Myklebust 12430445f92cSTrond Myklebust xprt_disconnect_done(xprt); 1244fe315e76SChuck Lever } 1245fe315e76SChuck Lever 1246fe315e76SChuck Lever /** 1247fe315e76SChuck Lever * xs_close - close a socket 1248fe315e76SChuck Lever * @xprt: transport 1249fe315e76SChuck Lever * 1250fe315e76SChuck Lever * This is used when all requests are complete; ie, no DRC state remains 1251fe315e76SChuck Lever * on the server we want to save. 1252f75e6745STrond Myklebust * 1253f75e6745STrond Myklebust * The caller _must_ be holding XPRT_LOCKED in order to avoid issues with 1254f75e6745STrond Myklebust * xs_reset_transport() zeroing the socket from underneath a writer. 1255fe315e76SChuck Lever */ 1256fe315e76SChuck Lever static void xs_close(struct rpc_xprt *xprt) 1257fe315e76SChuck Lever { 1258fe315e76SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1259fe315e76SChuck Lever 1260fe315e76SChuck Lever dprintk("RPC: xs_close xprt %p\n", xprt); 1261fe315e76SChuck Lever 1262fe315e76SChuck Lever xs_reset_transport(transport); 126361d0a8e6SNeil Brown xprt->reestablish_timeout = 0; 1264a246b010SChuck Lever } 1265a246b010SChuck Lever 12664a068258SChuck Lever static void xs_inject_disconnect(struct rpc_xprt *xprt) 12674a068258SChuck Lever { 12684a068258SChuck Lever dprintk("RPC: injecting transport disconnect on xprt=%p\n", 12694a068258SChuck Lever xprt); 12704a068258SChuck Lever xprt_disconnect_done(xprt); 12714a068258SChuck Lever } 12724a068258SChuck Lever 1273315f3812SKinglong Mee static void xs_xprt_free(struct rpc_xprt *xprt) 1274315f3812SKinglong Mee { 1275315f3812SKinglong Mee xs_free_peer_addresses(xprt); 1276315f3812SKinglong Mee xprt_free(xprt); 1277315f3812SKinglong Mee } 1278315f3812SKinglong Mee 12799903cd1cSChuck Lever /** 12809903cd1cSChuck Lever * xs_destroy - prepare to shutdown a transport 12819903cd1cSChuck Lever * @xprt: doomed transport 12829903cd1cSChuck Lever * 12839903cd1cSChuck Lever */ 12849903cd1cSChuck Lever static void xs_destroy(struct rpc_xprt *xprt) 1285a246b010SChuck Lever { 128603c78827STrond Myklebust struct sock_xprt *transport = container_of(xprt, 128703c78827STrond Myklebust struct sock_xprt, xprt); 12889903cd1cSChuck Lever dprintk("RPC: xs_destroy xprt %p\n", xprt); 12899903cd1cSChuck Lever 129003c78827STrond Myklebust cancel_delayed_work_sync(&transport->connect_worker); 1291a1311d87STrond Myklebust xs_close(xprt); 1292edc1b01cSTrond Myklebust cancel_work_sync(&transport->recv_worker); 1293315f3812SKinglong Mee xs_xprt_free(xprt); 1294a1311d87STrond Myklebust module_put(THIS_MODULE); 1295a246b010SChuck Lever } 1296a246b010SChuck Lever 12979903cd1cSChuck Lever /** 1298f9b2ee71STrond Myklebust * xs_udp_data_read_skb - receive callback for UDP sockets 1299f9b2ee71STrond Myklebust * @xprt: transport 1300f9b2ee71STrond Myklebust * @sk: socket 1301f9b2ee71STrond Myklebust * @skb: skbuff 13029903cd1cSChuck Lever * 1303a246b010SChuck Lever */ 1304f9b2ee71STrond Myklebust static void xs_udp_data_read_skb(struct rpc_xprt *xprt, 1305f9b2ee71STrond Myklebust struct sock *sk, 1306f9b2ee71STrond Myklebust struct sk_buff *skb) 1307a246b010SChuck Lever { 1308a246b010SChuck Lever struct rpc_task *task; 1309a246b010SChuck Lever struct rpc_rqst *rovr; 1310f9b2ee71STrond Myklebust int repsize, copied; 1311d8ed029dSAlexey Dobriyan u32 _xid; 1312d8ed029dSAlexey Dobriyan __be32 *xp; 1313a246b010SChuck Lever 13141da8c681SWillem de Bruijn repsize = skb->len; 1315a246b010SChuck Lever if (repsize < 4) { 13169903cd1cSChuck Lever dprintk("RPC: impossible RPC reply size %d!\n", repsize); 1317f9b2ee71STrond Myklebust return; 1318a246b010SChuck Lever } 1319a246b010SChuck Lever 1320a246b010SChuck Lever /* Copy the XID from the skb... */ 13211da8c681SWillem de Bruijn xp = skb_header_pointer(skb, 0, sizeof(_xid), &_xid); 1322a246b010SChuck Lever if (xp == NULL) 1323f9b2ee71STrond Myklebust return; 1324a246b010SChuck Lever 1325a246b010SChuck Lever /* Look up and lock the request corresponding to the given XID */ 132675c84151STrond Myklebust spin_lock(&xprt->queue_lock); 1327a246b010SChuck Lever rovr = xprt_lookup_rqst(xprt, *xp); 1328a246b010SChuck Lever if (!rovr) 1329a246b010SChuck Lever goto out_unlock; 1330729749bbSTrond Myklebust xprt_pin_rqst(rovr); 1331ecd465eeSChuck Lever xprt_update_rtt(rovr->rq_task); 133275c84151STrond Myklebust spin_unlock(&xprt->queue_lock); 1333a246b010SChuck Lever task = rovr->rq_task; 1334a246b010SChuck Lever 1335a246b010SChuck Lever if ((copied = rovr->rq_private_buf.buflen) > repsize) 1336a246b010SChuck Lever copied = repsize; 1337a246b010SChuck Lever 1338a246b010SChuck Lever /* Suck it into the iovec, verify checksum if not done by hw. */ 13391781f7f5SHerbert Xu if (csum_partial_copy_to_xdr(&rovr->rq_private_buf, skb)) { 134075c84151STrond Myklebust spin_lock(&xprt->queue_lock); 13410afa6b44STrond Myklebust __UDPX_INC_STATS(sk, UDP_MIB_INERRORS); 1342729749bbSTrond Myklebust goto out_unpin; 13431781f7f5SHerbert Xu } 13441781f7f5SHerbert Xu 1345a246b010SChuck Lever 1346729749bbSTrond Myklebust spin_lock_bh(&xprt->transport_lock); 13476a24dfb6STrond Myklebust xprt_adjust_cwnd(xprt, task, copied); 1348ce7c252aSTrond Myklebust spin_unlock_bh(&xprt->transport_lock); 134975c84151STrond Myklebust spin_lock(&xprt->queue_lock); 13501570c1e4SChuck Lever xprt_complete_rqst(task, copied); 13510afa6b44STrond Myklebust __UDPX_INC_STATS(sk, UDP_MIB_INDATAGRAMS); 1352729749bbSTrond Myklebust out_unpin: 1353729749bbSTrond Myklebust xprt_unpin_rqst(rovr); 1354a246b010SChuck Lever out_unlock: 135575c84151STrond Myklebust spin_unlock(&xprt->queue_lock); 1356f9b2ee71STrond Myklebust } 1357f9b2ee71STrond Myklebust 1358f9b2ee71STrond Myklebust static void xs_udp_data_receive(struct sock_xprt *transport) 1359f9b2ee71STrond Myklebust { 1360f9b2ee71STrond Myklebust struct sk_buff *skb; 1361f9b2ee71STrond Myklebust struct sock *sk; 1362f9b2ee71STrond Myklebust int err; 1363f9b2ee71STrond Myklebust 1364f9b2ee71STrond Myklebust mutex_lock(&transport->recv_mutex); 1365dfcf0380STrond Myklebust clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state); 1366f9b2ee71STrond Myklebust sk = transport->inet; 1367f9b2ee71STrond Myklebust if (sk == NULL) 1368f9b2ee71STrond Myklebust goto out; 1369f9b2ee71STrond Myklebust for (;;) { 13707c13f97fSPaolo Abeni skb = skb_recv_udp(sk, 0, 1, &err); 13714f546149STrond Myklebust if (skb == NULL) 13724f546149STrond Myklebust break; 1373f9b2ee71STrond Myklebust xs_udp_data_read_skb(&transport->xprt, sk, skb); 1374850cbaddSPaolo Abeni consume_skb(skb); 13750af3442aSTrond Myklebust cond_resched(); 1376f9b2ee71STrond Myklebust } 1377a246b010SChuck Lever out: 1378f9b2ee71STrond Myklebust mutex_unlock(&transport->recv_mutex); 1379f9b2ee71STrond Myklebust } 1380f9b2ee71STrond Myklebust 1381f9b2ee71STrond Myklebust static void xs_udp_data_receive_workfn(struct work_struct *work) 1382f9b2ee71STrond Myklebust { 1383f9b2ee71STrond Myklebust struct sock_xprt *transport = 1384f9b2ee71STrond Myklebust container_of(work, struct sock_xprt, recv_worker); 1385*a1231fdaSTrond Myklebust unsigned int pflags = memalloc_nofs_save(); 1386*a1231fdaSTrond Myklebust 1387f9b2ee71STrond Myklebust xs_udp_data_receive(transport); 1388*a1231fdaSTrond Myklebust memalloc_nofs_restore(pflags); 1389f9b2ee71STrond Myklebust } 1390f9b2ee71STrond Myklebust 1391f9b2ee71STrond Myklebust /** 1392f9b2ee71STrond Myklebust * xs_data_ready - "data ready" callback for UDP sockets 1393f9b2ee71STrond Myklebust * @sk: socket with data to read 1394f9b2ee71STrond Myklebust * 1395f9b2ee71STrond Myklebust */ 1396f9b2ee71STrond Myklebust static void xs_data_ready(struct sock *sk) 1397f9b2ee71STrond Myklebust { 1398f9b2ee71STrond Myklebust struct rpc_xprt *xprt; 1399f9b2ee71STrond Myklebust 1400f9b2ee71STrond Myklebust read_lock_bh(&sk->sk_callback_lock); 1401f9b2ee71STrond Myklebust dprintk("RPC: xs_data_ready...\n"); 1402f9b2ee71STrond Myklebust xprt = xprt_from_sock(sk); 1403f9b2ee71STrond Myklebust if (xprt != NULL) { 1404f9b2ee71STrond Myklebust struct sock_xprt *transport = container_of(xprt, 1405f9b2ee71STrond Myklebust struct sock_xprt, xprt); 14065157b956STrond Myklebust transport->old_data_ready(sk); 14075157b956STrond Myklebust /* Any data means we had a useful conversation, so 14085157b956STrond Myklebust * then we don't need to delay the next reconnect 14095157b956STrond Myklebust */ 14105157b956STrond Myklebust if (xprt->reestablish_timeout) 14115157b956STrond Myklebust xprt->reestablish_timeout = 0; 141242d42a5bSTrond Myklebust if (!test_and_set_bit(XPRT_SOCK_DATA_READY, &transport->sock_state)) 141340a5f1b1STrond Myklebust queue_work(xprtiod_workqueue, &transport->recv_worker); 1414f9b2ee71STrond Myklebust } 1415f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1416a246b010SChuck Lever } 1417a246b010SChuck Lever 1418a519fc7aSTrond Myklebust /* 1419a519fc7aSTrond Myklebust * Helper function to force a TCP close if the server is sending 1420a519fc7aSTrond Myklebust * junk and/or it has put us in CLOSE_WAIT 1421a519fc7aSTrond Myklebust */ 1422a519fc7aSTrond Myklebust static void xs_tcp_force_close(struct rpc_xprt *xprt) 1423a519fc7aSTrond Myklebust { 1424a519fc7aSTrond Myklebust xprt_force_disconnect(xprt); 1425a519fc7aSTrond Myklebust } 1426a519fc7aSTrond Myklebust 14279e00abc3STrond Myklebust #if defined(CONFIG_SUNRPC_BACKCHANNEL) 14286b26cc8cSChuck Lever static size_t xs_tcp_bc_maxpayload(struct rpc_xprt *xprt) 14296b26cc8cSChuck Lever { 14306b26cc8cSChuck Lever return PAGE_SIZE; 14316b26cc8cSChuck Lever } 14329e00abc3STrond Myklebust #endif /* CONFIG_SUNRPC_BACKCHANNEL */ 143344b98efdSRicardo Labiaga 14349903cd1cSChuck Lever /** 14359903cd1cSChuck Lever * xs_tcp_state_change - callback to handle TCP socket state changes 14369903cd1cSChuck Lever * @sk: socket whose state has changed 14379903cd1cSChuck Lever * 14389903cd1cSChuck Lever */ 14399903cd1cSChuck Lever static void xs_tcp_state_change(struct sock *sk) 1440a246b010SChuck Lever { 1441a246b010SChuck Lever struct rpc_xprt *xprt; 14420fdea1e8STrond Myklebust struct sock_xprt *transport; 1443a246b010SChuck Lever 1444f064af1eSEric Dumazet read_lock_bh(&sk->sk_callback_lock); 1445a246b010SChuck Lever if (!(xprt = xprt_from_sock(sk))) 1446a246b010SChuck Lever goto out; 14479903cd1cSChuck Lever dprintk("RPC: xs_tcp_state_change client %p...\n", xprt); 1448669502ffSAndy Chittenden dprintk("RPC: state %x conn %d dead %d zapped %d sk_shutdown %d\n", 1449a246b010SChuck Lever sk->sk_state, xprt_connected(xprt), 1450a246b010SChuck Lever sock_flag(sk, SOCK_DEAD), 1451669502ffSAndy Chittenden sock_flag(sk, SOCK_ZAPPED), 1452669502ffSAndy Chittenden sk->sk_shutdown); 1453a246b010SChuck Lever 14540fdea1e8STrond Myklebust transport = container_of(xprt, struct sock_xprt, xprt); 145540b5ea0cSTrond Myklebust trace_rpc_socket_state_change(xprt, sk->sk_socket); 1456a246b010SChuck Lever switch (sk->sk_state) { 1457a246b010SChuck Lever case TCP_ESTABLISHED: 1458f064af1eSEric Dumazet spin_lock(&xprt->transport_lock); 1459a246b010SChuck Lever if (!xprt_test_and_set_connected(xprt)) { 14608b71798cSTrond Myklebust xprt->connect_cookie++; 14610fdea1e8STrond Myklebust clear_bit(XPRT_SOCK_CONNECTING, &transport->sock_state); 14620fdea1e8STrond Myklebust xprt_clear_connecting(xprt); 146351971139SChuck Lever 14643968a8a5SChuck Lever xprt->stat.connect_count++; 14653968a8a5SChuck Lever xprt->stat.connect_time += (long)jiffies - 14663968a8a5SChuck Lever xprt->stat.connect_start; 14672a491991STrond Myklebust xprt_wake_pending_tasks(xprt, -EAGAIN); 1468a246b010SChuck Lever } 1469f064af1eSEric Dumazet spin_unlock(&xprt->transport_lock); 1470a246b010SChuck Lever break; 14713b948ae5STrond Myklebust case TCP_FIN_WAIT1: 14723b948ae5STrond Myklebust /* The client initiated a shutdown of the socket */ 14737c1d71cfSTrond Myklebust xprt->connect_cookie++; 1474663b8858STrond Myklebust xprt->reestablish_timeout = 0; 14753b948ae5STrond Myklebust set_bit(XPRT_CLOSING, &xprt->state); 14764e857c58SPeter Zijlstra smp_mb__before_atomic(); 14773b948ae5STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state); 1478ef803670STrond Myklebust clear_bit(XPRT_CLOSE_WAIT, &xprt->state); 14794e857c58SPeter Zijlstra smp_mb__after_atomic(); 1480a246b010SChuck Lever break; 1481632e3bdcSTrond Myklebust case TCP_CLOSE_WAIT: 14823b948ae5STrond Myklebust /* The server initiated a shutdown of the socket */ 14837c1d71cfSTrond Myklebust xprt->connect_cookie++; 1484d0bea455STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state); 1485a519fc7aSTrond Myklebust xs_tcp_force_close(xprt); 1486e9d47639SGustavo A. R. Silva /* fall through */ 1487663b8858STrond Myklebust case TCP_CLOSING: 1488663b8858STrond Myklebust /* 1489663b8858STrond Myklebust * If the server closed down the connection, make sure that 1490663b8858STrond Myklebust * we back off before reconnecting 1491663b8858STrond Myklebust */ 1492663b8858STrond Myklebust if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO) 1493663b8858STrond Myklebust xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 14943b948ae5STrond Myklebust break; 14953b948ae5STrond Myklebust case TCP_LAST_ACK: 1496670f9457STrond Myklebust set_bit(XPRT_CLOSING, &xprt->state); 14974e857c58SPeter Zijlstra smp_mb__before_atomic(); 14983b948ae5STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state); 14994e857c58SPeter Zijlstra smp_mb__after_atomic(); 15003b948ae5STrond Myklebust break; 15013b948ae5STrond Myklebust case TCP_CLOSE: 15020fdea1e8STrond Myklebust if (test_and_clear_bit(XPRT_SOCK_CONNECTING, 15030fdea1e8STrond Myklebust &transport->sock_state)) 15040fdea1e8STrond Myklebust xprt_clear_connecting(xprt); 15059b30889cSTrond Myklebust clear_bit(XPRT_CLOSING, &xprt->state); 15069b30889cSTrond Myklebust /* Trigger the socket release */ 15079b30889cSTrond Myklebust xs_tcp_force_close(xprt); 1508a246b010SChuck Lever } 1509a246b010SChuck Lever out: 1510f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1511a246b010SChuck Lever } 1512a246b010SChuck Lever 15131f0fa154SIlpo Järvinen static void xs_write_space(struct sock *sk) 15141f0fa154SIlpo Järvinen { 151513331a55STrond Myklebust struct socket_wq *wq; 15161f0fa154SIlpo Järvinen struct rpc_xprt *xprt; 15171f0fa154SIlpo Järvinen 151813331a55STrond Myklebust if (!sk->sk_socket) 15191f0fa154SIlpo Järvinen return; 152013331a55STrond Myklebust clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 15211f0fa154SIlpo Järvinen 15221f0fa154SIlpo Järvinen if (unlikely(!(xprt = xprt_from_sock(sk)))) 15231f0fa154SIlpo Järvinen return; 152413331a55STrond Myklebust rcu_read_lock(); 152513331a55STrond Myklebust wq = rcu_dereference(sk->sk_wq); 152613331a55STrond Myklebust if (!wq || test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &wq->flags) == 0) 152713331a55STrond Myklebust goto out; 15281f0fa154SIlpo Järvinen 1529c544577dSTrond Myklebust if (xprt_write_space(xprt)) 1530c544577dSTrond Myklebust sk->sk_write_pending--; 153113331a55STrond Myklebust out: 153213331a55STrond Myklebust rcu_read_unlock(); 15331f0fa154SIlpo Järvinen } 15341f0fa154SIlpo Järvinen 15352a9e1cfaSTrond Myklebust /** 1536c7b2cae8SChuck Lever * xs_udp_write_space - callback invoked when socket buffer space 1537c7b2cae8SChuck Lever * becomes available 15389903cd1cSChuck Lever * @sk: socket whose state has changed 15399903cd1cSChuck Lever * 1540a246b010SChuck Lever * Called when more output buffer space is available for this socket. 1541a246b010SChuck Lever * We try not to wake our writers until they can make "significant" 1542c7b2cae8SChuck Lever * progress, otherwise we'll waste resources thrashing kernel_sendmsg 1543a246b010SChuck Lever * with a bunch of small requests. 1544a246b010SChuck Lever */ 1545c7b2cae8SChuck Lever static void xs_udp_write_space(struct sock *sk) 1546a246b010SChuck Lever { 1547f064af1eSEric Dumazet read_lock_bh(&sk->sk_callback_lock); 1548c7b2cae8SChuck Lever 1549c7b2cae8SChuck Lever /* from net/core/sock.c:sock_def_write_space */ 15501f0fa154SIlpo Järvinen if (sock_writeable(sk)) 15511f0fa154SIlpo Järvinen xs_write_space(sk); 1552c7b2cae8SChuck Lever 1553f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1554c7b2cae8SChuck Lever } 1555c7b2cae8SChuck Lever 1556c7b2cae8SChuck Lever /** 1557c7b2cae8SChuck Lever * xs_tcp_write_space - callback invoked when socket buffer space 1558c7b2cae8SChuck Lever * becomes available 1559c7b2cae8SChuck Lever * @sk: socket whose state has changed 1560c7b2cae8SChuck Lever * 1561c7b2cae8SChuck Lever * Called when more output buffer space is available for this socket. 1562c7b2cae8SChuck Lever * We try not to wake our writers until they can make "significant" 1563c7b2cae8SChuck Lever * progress, otherwise we'll waste resources thrashing kernel_sendmsg 1564c7b2cae8SChuck Lever * with a bunch of small requests. 1565c7b2cae8SChuck Lever */ 1566c7b2cae8SChuck Lever static void xs_tcp_write_space(struct sock *sk) 1567c7b2cae8SChuck Lever { 1568f064af1eSEric Dumazet read_lock_bh(&sk->sk_callback_lock); 1569c7b2cae8SChuck Lever 1570c7b2cae8SChuck Lever /* from net/core/stream.c:sk_stream_write_space */ 157164dc6130SEric Dumazet if (sk_stream_is_writeable(sk)) 15721f0fa154SIlpo Järvinen xs_write_space(sk); 1573c7b2cae8SChuck Lever 1574f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1575a246b010SChuck Lever } 1576a246b010SChuck Lever 1577470056c2SChuck Lever static void xs_udp_do_set_buffer_size(struct rpc_xprt *xprt) 1578a246b010SChuck Lever { 1579ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1580ee0ac0c2SChuck Lever struct sock *sk = transport->inet; 1581a246b010SChuck Lever 15827c6e066eSChuck Lever if (transport->rcvsize) { 1583a246b010SChuck Lever sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 15847c6e066eSChuck Lever sk->sk_rcvbuf = transport->rcvsize * xprt->max_reqs * 2; 1585a246b010SChuck Lever } 15867c6e066eSChuck Lever if (transport->sndsize) { 1587a246b010SChuck Lever sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 15887c6e066eSChuck Lever sk->sk_sndbuf = transport->sndsize * xprt->max_reqs * 2; 1589a246b010SChuck Lever sk->sk_write_space(sk); 1590a246b010SChuck Lever } 1591a246b010SChuck Lever } 1592a246b010SChuck Lever 159343118c29SChuck Lever /** 1594470056c2SChuck Lever * xs_udp_set_buffer_size - set send and receive limits 159543118c29SChuck Lever * @xprt: generic transport 1596470056c2SChuck Lever * @sndsize: requested size of send buffer, in bytes 1597470056c2SChuck Lever * @rcvsize: requested size of receive buffer, in bytes 159843118c29SChuck Lever * 1599470056c2SChuck Lever * Set socket send and receive buffer size limits. 160043118c29SChuck Lever */ 1601470056c2SChuck Lever static void xs_udp_set_buffer_size(struct rpc_xprt *xprt, size_t sndsize, size_t rcvsize) 160243118c29SChuck Lever { 16037c6e066eSChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 16047c6e066eSChuck Lever 16057c6e066eSChuck Lever transport->sndsize = 0; 1606470056c2SChuck Lever if (sndsize) 16077c6e066eSChuck Lever transport->sndsize = sndsize + 1024; 16087c6e066eSChuck Lever transport->rcvsize = 0; 1609470056c2SChuck Lever if (rcvsize) 16107c6e066eSChuck Lever transport->rcvsize = rcvsize + 1024; 1611470056c2SChuck Lever 1612470056c2SChuck Lever xs_udp_do_set_buffer_size(xprt); 161343118c29SChuck Lever } 161443118c29SChuck Lever 161546c0ee8bSChuck Lever /** 161646c0ee8bSChuck Lever * xs_udp_timer - called when a retransmit timeout occurs on a UDP transport 1617acf0a39fSChuck Lever * @xprt: controlling transport 161846c0ee8bSChuck Lever * @task: task that timed out 161946c0ee8bSChuck Lever * 162046c0ee8bSChuck Lever * Adjust the congestion window after a retransmit timeout has occurred. 162146c0ee8bSChuck Lever */ 16226a24dfb6STrond Myklebust static void xs_udp_timer(struct rpc_xprt *xprt, struct rpc_task *task) 162346c0ee8bSChuck Lever { 1624b977b644SChuck Lever spin_lock_bh(&xprt->transport_lock); 16256a24dfb6STrond Myklebust xprt_adjust_cwnd(xprt, task, -ETIMEDOUT); 1626b977b644SChuck Lever spin_unlock_bh(&xprt->transport_lock); 162746c0ee8bSChuck Lever } 162846c0ee8bSChuck Lever 1629826799e6SJ. Bruce Fields static int xs_get_random_port(void) 1630b85d8806SChuck Lever { 1631826799e6SJ. Bruce Fields unsigned short min = xprt_min_resvport, max = xprt_max_resvport; 1632826799e6SJ. Bruce Fields unsigned short range; 1633826799e6SJ. Bruce Fields unsigned short rand; 1634826799e6SJ. Bruce Fields 1635826799e6SJ. Bruce Fields if (max < min) 1636826799e6SJ. Bruce Fields return -EADDRINUSE; 1637826799e6SJ. Bruce Fields range = max - min + 1; 1638826799e6SJ. Bruce Fields rand = (unsigned short) prandom_u32() % range; 1639826799e6SJ. Bruce Fields return rand + min; 1640b85d8806SChuck Lever } 1641b85d8806SChuck Lever 164292200412SChuck Lever /** 16434dda9c8aSTrond Myklebust * xs_set_reuseaddr_port - set the socket's port and address reuse options 16444dda9c8aSTrond Myklebust * @sock: socket 16454dda9c8aSTrond Myklebust * 16464dda9c8aSTrond Myklebust * Note that this function has to be called on all sockets that share the 16474dda9c8aSTrond Myklebust * same port, and it must be called before binding. 16484dda9c8aSTrond Myklebust */ 16494dda9c8aSTrond Myklebust static void xs_sock_set_reuseport(struct socket *sock) 16504dda9c8aSTrond Myklebust { 1651402e23b4STrond Myklebust int opt = 1; 16524dda9c8aSTrond Myklebust 1653402e23b4STrond Myklebust kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, 1654402e23b4STrond Myklebust (char *)&opt, sizeof(opt)); 16554dda9c8aSTrond Myklebust } 16564dda9c8aSTrond Myklebust 16574dda9c8aSTrond Myklebust static unsigned short xs_sock_getport(struct socket *sock) 16584dda9c8aSTrond Myklebust { 16594dda9c8aSTrond Myklebust struct sockaddr_storage buf; 16604dda9c8aSTrond Myklebust unsigned short port = 0; 16614dda9c8aSTrond Myklebust 16629b2c45d4SDenys Vlasenko if (kernel_getsockname(sock, (struct sockaddr *)&buf) < 0) 16634dda9c8aSTrond Myklebust goto out; 16644dda9c8aSTrond Myklebust switch (buf.ss_family) { 16654dda9c8aSTrond Myklebust case AF_INET6: 16664dda9c8aSTrond Myklebust port = ntohs(((struct sockaddr_in6 *)&buf)->sin6_port); 16674dda9c8aSTrond Myklebust break; 16684dda9c8aSTrond Myklebust case AF_INET: 16694dda9c8aSTrond Myklebust port = ntohs(((struct sockaddr_in *)&buf)->sin_port); 16704dda9c8aSTrond Myklebust } 16714dda9c8aSTrond Myklebust out: 16724dda9c8aSTrond Myklebust return port; 16734dda9c8aSTrond Myklebust } 16744dda9c8aSTrond Myklebust 16754dda9c8aSTrond Myklebust /** 167692200412SChuck Lever * xs_set_port - reset the port number in the remote endpoint address 167792200412SChuck Lever * @xprt: generic transport 167892200412SChuck Lever * @port: new port number 167992200412SChuck Lever * 168092200412SChuck Lever */ 168192200412SChuck Lever static void xs_set_port(struct rpc_xprt *xprt, unsigned short port) 168292200412SChuck Lever { 168392200412SChuck Lever dprintk("RPC: setting port for xprt %p to %u\n", xprt, port); 1684c4efcb1dSChuck Lever 16859dc3b095SChuck Lever rpc_set_port(xs_addr(xprt), port); 16869dc3b095SChuck Lever xs_update_peer_port(xprt); 168792200412SChuck Lever } 168892200412SChuck Lever 16894dda9c8aSTrond Myklebust static void xs_set_srcport(struct sock_xprt *transport, struct socket *sock) 16904dda9c8aSTrond Myklebust { 16914dda9c8aSTrond Myklebust if (transport->srcport == 0) 16924dda9c8aSTrond Myklebust transport->srcport = xs_sock_getport(sock); 16934dda9c8aSTrond Myklebust } 16944dda9c8aSTrond Myklebust 1695826799e6SJ. Bruce Fields static int xs_get_srcport(struct sock_xprt *transport) 169667a391d7STrond Myklebust { 1697826799e6SJ. Bruce Fields int port = transport->srcport; 169867a391d7STrond Myklebust 169967a391d7STrond Myklebust if (port == 0 && transport->xprt.resvport) 170067a391d7STrond Myklebust port = xs_get_random_port(); 170167a391d7STrond Myklebust return port; 170267a391d7STrond Myklebust } 170367a391d7STrond Myklebust 1704baaf4e48SPavel Emelyanov static unsigned short xs_next_srcport(struct sock_xprt *transport, unsigned short port) 170567a391d7STrond Myklebust { 1706fbfffbd5SChuck Lever if (transport->srcport != 0) 1707fbfffbd5SChuck Lever transport->srcport = 0; 170867a391d7STrond Myklebust if (!transport->xprt.resvport) 170967a391d7STrond Myklebust return 0; 171067a391d7STrond Myklebust if (port <= xprt_min_resvport || port > xprt_max_resvport) 171167a391d7STrond Myklebust return xprt_max_resvport; 171267a391d7STrond Myklebust return --port; 171367a391d7STrond Myklebust } 1714beb59b68SPavel Emelyanov static int xs_bind(struct sock_xprt *transport, struct socket *sock) 1715a246b010SChuck Lever { 1716beb59b68SPavel Emelyanov struct sockaddr_storage myaddr; 171767a391d7STrond Myklebust int err, nloop = 0; 1718826799e6SJ. Bruce Fields int port = xs_get_srcport(transport); 171967a391d7STrond Myklebust unsigned short last; 1720a246b010SChuck Lever 17210f7a622cSChris Perl /* 17220f7a622cSChris Perl * If we are asking for any ephemeral port (i.e. port == 0 && 17230f7a622cSChris Perl * transport->xprt.resvport == 0), don't bind. Let the local 17240f7a622cSChris Perl * port selection happen implicitly when the socket is used 17250f7a622cSChris Perl * (for example at connect time). 17260f7a622cSChris Perl * 17270f7a622cSChris Perl * This ensures that we can continue to establish TCP 17280f7a622cSChris Perl * connections even when all local ephemeral ports are already 17290f7a622cSChris Perl * a part of some TCP connection. This makes no difference 17300f7a622cSChris Perl * for UDP sockets, but also doens't harm them. 17310f7a622cSChris Perl * 17320f7a622cSChris Perl * If we're asking for any reserved port (i.e. port == 0 && 17330f7a622cSChris Perl * transport->xprt.resvport == 1) xs_get_srcport above will 17340f7a622cSChris Perl * ensure that port is non-zero and we will bind as needed. 17350f7a622cSChris Perl */ 1736826799e6SJ. Bruce Fields if (port <= 0) 1737826799e6SJ. Bruce Fields return port; 17380f7a622cSChris Perl 1739beb59b68SPavel Emelyanov memcpy(&myaddr, &transport->srcaddr, transport->xprt.addrlen); 1740a246b010SChuck Lever do { 1741beb59b68SPavel Emelyanov rpc_set_port((struct sockaddr *)&myaddr, port); 1742e6242e92SSridhar Samudrala err = kernel_bind(sock, (struct sockaddr *)&myaddr, 1743beb59b68SPavel Emelyanov transport->xprt.addrlen); 1744a246b010SChuck Lever if (err == 0) { 1745fbfffbd5SChuck Lever transport->srcport = port; 1746d3bc9a1dSFrank van Maarseveen break; 1747a246b010SChuck Lever } 174867a391d7STrond Myklebust last = port; 1749baaf4e48SPavel Emelyanov port = xs_next_srcport(transport, port); 175067a391d7STrond Myklebust if (port > last) 175167a391d7STrond Myklebust nloop++; 175267a391d7STrond Myklebust } while (err == -EADDRINUSE && nloop != 2); 1753beb59b68SPavel Emelyanov 17544232e863SChuck Lever if (myaddr.ss_family == AF_INET) 1755beb59b68SPavel Emelyanov dprintk("RPC: %s %pI4:%u: %s (%d)\n", __func__, 1756beb59b68SPavel Emelyanov &((struct sockaddr_in *)&myaddr)->sin_addr, 1757beb59b68SPavel Emelyanov port, err ? "failed" : "ok", err); 1758beb59b68SPavel Emelyanov else 1759beb59b68SPavel Emelyanov dprintk("RPC: %s %pI6:%u: %s (%d)\n", __func__, 1760beb59b68SPavel Emelyanov &((struct sockaddr_in6 *)&myaddr)->sin6_addr, 17617dc753f0SChuck Lever port, err ? "failed" : "ok", err); 1762a246b010SChuck Lever return err; 1763a246b010SChuck Lever } 1764a246b010SChuck Lever 1765176e21eeSChuck Lever /* 1766176e21eeSChuck Lever * We don't support autobind on AF_LOCAL sockets 1767176e21eeSChuck Lever */ 1768176e21eeSChuck Lever static void xs_local_rpcbind(struct rpc_task *task) 1769176e21eeSChuck Lever { 1770fb43d172STrond Myklebust xprt_set_bound(task->tk_xprt); 1771176e21eeSChuck Lever } 1772176e21eeSChuck Lever 1773176e21eeSChuck Lever static void xs_local_set_port(struct rpc_xprt *xprt, unsigned short port) 1774176e21eeSChuck Lever { 1775176e21eeSChuck Lever } 1776a246b010SChuck Lever 1777ed07536eSPeter Zijlstra #ifdef CONFIG_DEBUG_LOCK_ALLOC 1778ed07536eSPeter Zijlstra static struct lock_class_key xs_key[2]; 1779ed07536eSPeter Zijlstra static struct lock_class_key xs_slock_key[2]; 1780ed07536eSPeter Zijlstra 1781176e21eeSChuck Lever static inline void xs_reclassify_socketu(struct socket *sock) 1782176e21eeSChuck Lever { 1783176e21eeSChuck Lever struct sock *sk = sock->sk; 1784176e21eeSChuck Lever 1785176e21eeSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_LOCAL-RPC", 1786176e21eeSChuck Lever &xs_slock_key[1], "sk_lock-AF_LOCAL-RPC", &xs_key[1]); 1787176e21eeSChuck Lever } 1788176e21eeSChuck Lever 17898945ee5eSChuck Lever static inline void xs_reclassify_socket4(struct socket *sock) 1790ed07536eSPeter Zijlstra { 1791ed07536eSPeter Zijlstra struct sock *sk = sock->sk; 17928945ee5eSChuck Lever 17938945ee5eSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_INET-RPC", 17948945ee5eSChuck Lever &xs_slock_key[0], "sk_lock-AF_INET-RPC", &xs_key[0]); 1795ed07536eSPeter Zijlstra } 17968945ee5eSChuck Lever 17978945ee5eSChuck Lever static inline void xs_reclassify_socket6(struct socket *sock) 17988945ee5eSChuck Lever { 17998945ee5eSChuck Lever struct sock *sk = sock->sk; 18008945ee5eSChuck Lever 18018945ee5eSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_INET6-RPC", 18028945ee5eSChuck Lever &xs_slock_key[1], "sk_lock-AF_INET6-RPC", &xs_key[1]); 1803ed07536eSPeter Zijlstra } 18046bc9638aSPavel Emelyanov 18056bc9638aSPavel Emelyanov static inline void xs_reclassify_socket(int family, struct socket *sock) 18066bc9638aSPavel Emelyanov { 1807fafc4e1eSHannes Frederic Sowa if (WARN_ON_ONCE(!sock_allow_reclassification(sock->sk))) 18081b7a1819SWeston Andros Adamson return; 18091b7a1819SWeston Andros Adamson 18104232e863SChuck Lever switch (family) { 1811176e21eeSChuck Lever case AF_LOCAL: 1812176e21eeSChuck Lever xs_reclassify_socketu(sock); 1813176e21eeSChuck Lever break; 18144232e863SChuck Lever case AF_INET: 18156bc9638aSPavel Emelyanov xs_reclassify_socket4(sock); 18164232e863SChuck Lever break; 18174232e863SChuck Lever case AF_INET6: 18186bc9638aSPavel Emelyanov xs_reclassify_socket6(sock); 18194232e863SChuck Lever break; 18204232e863SChuck Lever } 18216bc9638aSPavel Emelyanov } 1822ed07536eSPeter Zijlstra #else 18236bc9638aSPavel Emelyanov static inline void xs_reclassify_socket(int family, struct socket *sock) 18246bc9638aSPavel Emelyanov { 18256bc9638aSPavel Emelyanov } 1826ed07536eSPeter Zijlstra #endif 1827ed07536eSPeter Zijlstra 182893dc41bdSNeilBrown static void xs_dummy_setup_socket(struct work_struct *work) 182993dc41bdSNeilBrown { 183093dc41bdSNeilBrown } 183193dc41bdSNeilBrown 18326bc9638aSPavel Emelyanov static struct socket *xs_create_sock(struct rpc_xprt *xprt, 18334dda9c8aSTrond Myklebust struct sock_xprt *transport, int family, int type, 18344dda9c8aSTrond Myklebust int protocol, bool reuseport) 183522f79326SPavel Emelyanov { 183622f79326SPavel Emelyanov struct socket *sock; 183722f79326SPavel Emelyanov int err; 183822f79326SPavel Emelyanov 18396bc9638aSPavel Emelyanov err = __sock_create(xprt->xprt_net, family, type, protocol, &sock, 1); 184022f79326SPavel Emelyanov if (err < 0) { 184122f79326SPavel Emelyanov dprintk("RPC: can't create %d transport socket (%d).\n", 184222f79326SPavel Emelyanov protocol, -err); 184322f79326SPavel Emelyanov goto out; 184422f79326SPavel Emelyanov } 18456bc9638aSPavel Emelyanov xs_reclassify_socket(family, sock); 184622f79326SPavel Emelyanov 18474dda9c8aSTrond Myklebust if (reuseport) 18484dda9c8aSTrond Myklebust xs_sock_set_reuseport(sock); 18494dda9c8aSTrond Myklebust 18504cea288aSBen Hutchings err = xs_bind(transport, sock); 18514cea288aSBen Hutchings if (err) { 185222f79326SPavel Emelyanov sock_release(sock); 185322f79326SPavel Emelyanov goto out; 185422f79326SPavel Emelyanov } 185522f79326SPavel Emelyanov 185622f79326SPavel Emelyanov return sock; 185722f79326SPavel Emelyanov out: 185822f79326SPavel Emelyanov return ERR_PTR(err); 185922f79326SPavel Emelyanov } 186022f79326SPavel Emelyanov 1861176e21eeSChuck Lever static int xs_local_finish_connecting(struct rpc_xprt *xprt, 1862176e21eeSChuck Lever struct socket *sock) 1863176e21eeSChuck Lever { 1864176e21eeSChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, 1865176e21eeSChuck Lever xprt); 1866176e21eeSChuck Lever 1867176e21eeSChuck Lever if (!transport->inet) { 1868176e21eeSChuck Lever struct sock *sk = sock->sk; 1869176e21eeSChuck Lever 1870176e21eeSChuck Lever write_lock_bh(&sk->sk_callback_lock); 1871176e21eeSChuck Lever 1872176e21eeSChuck Lever xs_save_old_callbacks(transport, sk); 1873176e21eeSChuck Lever 1874176e21eeSChuck Lever sk->sk_user_data = xprt; 1875a2648094STrond Myklebust sk->sk_data_ready = xs_data_ready; 1876176e21eeSChuck Lever sk->sk_write_space = xs_udp_write_space; 1877b4411457SEric Dumazet sock_set_flag(sk, SOCK_FASYNC); 18782118071dSTrond Myklebust sk->sk_error_report = xs_error_report; 1879176e21eeSChuck Lever 1880176e21eeSChuck Lever xprt_clear_connected(xprt); 1881176e21eeSChuck Lever 1882176e21eeSChuck Lever /* Reset to new socket */ 1883176e21eeSChuck Lever transport->sock = sock; 1884176e21eeSChuck Lever transport->inet = sk; 1885176e21eeSChuck Lever 1886176e21eeSChuck Lever write_unlock_bh(&sk->sk_callback_lock); 1887176e21eeSChuck Lever } 1888176e21eeSChuck Lever 1889550aebfeSTrond Myklebust xs_stream_reset_connect(transport); 18906c7a64e5STrond Myklebust 1891176e21eeSChuck Lever return kernel_connect(sock, xs_addr(xprt), xprt->addrlen, 0); 1892176e21eeSChuck Lever } 1893176e21eeSChuck Lever 1894176e21eeSChuck Lever /** 1895176e21eeSChuck Lever * xs_local_setup_socket - create AF_LOCAL socket, connect to a local endpoint 1896176e21eeSChuck Lever * @transport: socket transport to connect 1897176e21eeSChuck Lever */ 1898dc107402SJ. Bruce Fields static int xs_local_setup_socket(struct sock_xprt *transport) 1899176e21eeSChuck Lever { 1900176e21eeSChuck Lever struct rpc_xprt *xprt = &transport->xprt; 1901176e21eeSChuck Lever struct socket *sock; 1902176e21eeSChuck Lever int status = -EIO; 1903176e21eeSChuck Lever 1904176e21eeSChuck Lever status = __sock_create(xprt->xprt_net, AF_LOCAL, 1905176e21eeSChuck Lever SOCK_STREAM, 0, &sock, 1); 1906176e21eeSChuck Lever if (status < 0) { 1907176e21eeSChuck Lever dprintk("RPC: can't create AF_LOCAL " 1908176e21eeSChuck Lever "transport socket (%d).\n", -status); 1909176e21eeSChuck Lever goto out; 1910176e21eeSChuck Lever } 1911d1358917SStefan Hajnoczi xs_reclassify_socket(AF_LOCAL, sock); 1912176e21eeSChuck Lever 1913176e21eeSChuck Lever dprintk("RPC: worker connecting xprt %p via AF_LOCAL to %s\n", 1914176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 1915176e21eeSChuck Lever 1916176e21eeSChuck Lever status = xs_local_finish_connecting(xprt, sock); 191740b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, status); 1918176e21eeSChuck Lever switch (status) { 1919176e21eeSChuck Lever case 0: 1920176e21eeSChuck Lever dprintk("RPC: xprt %p connected to %s\n", 1921176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 19223968a8a5SChuck Lever xprt->stat.connect_count++; 19233968a8a5SChuck Lever xprt->stat.connect_time += (long)jiffies - 19243968a8a5SChuck Lever xprt->stat.connect_start; 1925176e21eeSChuck Lever xprt_set_connected(xprt); 19263601c4a9STrond Myklebust case -ENOBUFS: 1927176e21eeSChuck Lever break; 1928176e21eeSChuck Lever case -ENOENT: 1929176e21eeSChuck Lever dprintk("RPC: xprt %p: socket %s does not exist\n", 1930176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 1931176e21eeSChuck Lever break; 19324a20a988STrond Myklebust case -ECONNREFUSED: 19334a20a988STrond Myklebust dprintk("RPC: xprt %p: connection refused for %s\n", 19344a20a988STrond Myklebust xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 19354a20a988STrond Myklebust break; 1936176e21eeSChuck Lever default: 1937176e21eeSChuck Lever printk(KERN_ERR "%s: unhandled error (%d) connecting to %s\n", 1938176e21eeSChuck Lever __func__, -status, 1939176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR]); 1940176e21eeSChuck Lever } 1941176e21eeSChuck Lever 1942176e21eeSChuck Lever out: 1943176e21eeSChuck Lever xprt_clear_connecting(xprt); 1944176e21eeSChuck Lever xprt_wake_pending_tasks(xprt, status); 1945dc107402SJ. Bruce Fields return status; 1946dc107402SJ. Bruce Fields } 1947dc107402SJ. Bruce Fields 1948b6669737SLinus Torvalds static void xs_local_connect(struct rpc_xprt *xprt, struct rpc_task *task) 1949dc107402SJ. Bruce Fields { 1950dc107402SJ. Bruce Fields struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1951dc107402SJ. Bruce Fields int ret; 1952dc107402SJ. Bruce Fields 1953dc107402SJ. Bruce Fields if (RPC_IS_ASYNC(task)) { 1954dc107402SJ. Bruce Fields /* 1955dc107402SJ. Bruce Fields * We want the AF_LOCAL connect to be resolved in the 1956dc107402SJ. Bruce Fields * filesystem namespace of the process making the rpc 1957dc107402SJ. Bruce Fields * call. Thus we connect synchronously. 1958dc107402SJ. Bruce Fields * 1959dc107402SJ. Bruce Fields * If we want to support asynchronous AF_LOCAL calls, 1960dc107402SJ. Bruce Fields * we'll need to figure out how to pass a namespace to 1961dc107402SJ. Bruce Fields * connect. 1962dc107402SJ. Bruce Fields */ 1963dc107402SJ. Bruce Fields rpc_exit(task, -ENOTCONN); 1964dc107402SJ. Bruce Fields return; 1965dc107402SJ. Bruce Fields } 1966dc107402SJ. Bruce Fields ret = xs_local_setup_socket(transport); 1967dc107402SJ. Bruce Fields if (ret && !RPC_IS_SOFTCONN(task)) 1968dc107402SJ. Bruce Fields msleep_interruptible(15000); 1969176e21eeSChuck Lever } 1970176e21eeSChuck Lever 19713c87ef6eSJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_SWAP) 1972d6e971d8SJeff Layton /* 1973d6e971d8SJeff Layton * Note that this should be called with XPRT_LOCKED held (or when we otherwise 1974d6e971d8SJeff Layton * know that we have exclusive access to the socket), to guard against 1975d6e971d8SJeff Layton * races with xs_reset_transport. 1976d6e971d8SJeff Layton */ 1977a564b8f0SMel Gorman static void xs_set_memalloc(struct rpc_xprt *xprt) 1978a564b8f0SMel Gorman { 1979a564b8f0SMel Gorman struct sock_xprt *transport = container_of(xprt, struct sock_xprt, 1980a564b8f0SMel Gorman xprt); 1981a564b8f0SMel Gorman 1982d6e971d8SJeff Layton /* 1983d6e971d8SJeff Layton * If there's no sock, then we have nothing to set. The 1984d6e971d8SJeff Layton * reconnecting process will get it for us. 1985d6e971d8SJeff Layton */ 1986d6e971d8SJeff Layton if (!transport->inet) 1987d6e971d8SJeff Layton return; 19888e228133SJeff Layton if (atomic_read(&xprt->swapper)) 1989a564b8f0SMel Gorman sk_set_memalloc(transport->inet); 1990a564b8f0SMel Gorman } 1991a564b8f0SMel Gorman 1992a564b8f0SMel Gorman /** 1993d67fa4d8SJeff Layton * xs_enable_swap - Tag this transport as being used for swap. 1994a564b8f0SMel Gorman * @xprt: transport to tag 1995a564b8f0SMel Gorman * 19968e228133SJeff Layton * Take a reference to this transport on behalf of the rpc_clnt, and 19978e228133SJeff Layton * optionally mark it for swapping if it wasn't already. 1998a564b8f0SMel Gorman */ 1999d67fa4d8SJeff Layton static int 2000d67fa4d8SJeff Layton xs_enable_swap(struct rpc_xprt *xprt) 2001a564b8f0SMel Gorman { 2002d6e971d8SJeff Layton struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt); 2003a564b8f0SMel Gorman 2004d6e971d8SJeff Layton if (atomic_inc_return(&xprt->swapper) != 1) 2005d6e971d8SJeff Layton return 0; 2006d6e971d8SJeff Layton if (wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE)) 2007d6e971d8SJeff Layton return -ERESTARTSYS; 2008d6e971d8SJeff Layton if (xs->inet) 2009d6e971d8SJeff Layton sk_set_memalloc(xs->inet); 2010d6e971d8SJeff Layton xprt_release_xprt(xprt, NULL); 20118e228133SJeff Layton return 0; 2012a564b8f0SMel Gorman } 2013a564b8f0SMel Gorman 20148e228133SJeff Layton /** 2015d67fa4d8SJeff Layton * xs_disable_swap - Untag this transport as being used for swap. 20168e228133SJeff Layton * @xprt: transport to tag 20178e228133SJeff Layton * 20188e228133SJeff Layton * Drop a "swapper" reference to this xprt on behalf of the rpc_clnt. If the 20198e228133SJeff Layton * swapper refcount goes to 0, untag the socket as a memalloc socket. 20208e228133SJeff Layton */ 2021d67fa4d8SJeff Layton static void 2022d67fa4d8SJeff Layton xs_disable_swap(struct rpc_xprt *xprt) 20238e228133SJeff Layton { 2024d6e971d8SJeff Layton struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt); 20258e228133SJeff Layton 2026d6e971d8SJeff Layton if (!atomic_dec_and_test(&xprt->swapper)) 2027d6e971d8SJeff Layton return; 2028d6e971d8SJeff Layton if (wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE)) 2029d6e971d8SJeff Layton return; 2030d6e971d8SJeff Layton if (xs->inet) 2031d6e971d8SJeff Layton sk_clear_memalloc(xs->inet); 2032d6e971d8SJeff Layton xprt_release_xprt(xprt, NULL); 2033a564b8f0SMel Gorman } 2034a564b8f0SMel Gorman #else 2035a564b8f0SMel Gorman static void xs_set_memalloc(struct rpc_xprt *xprt) 2036a564b8f0SMel Gorman { 2037a564b8f0SMel Gorman } 2038d67fa4d8SJeff Layton 2039d67fa4d8SJeff Layton static int 2040d67fa4d8SJeff Layton xs_enable_swap(struct rpc_xprt *xprt) 2041d67fa4d8SJeff Layton { 2042d67fa4d8SJeff Layton return -EINVAL; 2043d67fa4d8SJeff Layton } 2044d67fa4d8SJeff Layton 2045d67fa4d8SJeff Layton static void 2046d67fa4d8SJeff Layton xs_disable_swap(struct rpc_xprt *xprt) 2047d67fa4d8SJeff Layton { 2048d67fa4d8SJeff Layton } 2049a564b8f0SMel Gorman #endif 2050a564b8f0SMel Gorman 205116be2d20SChuck Lever static void xs_udp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock) 2052a246b010SChuck Lever { 205316be2d20SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2054edb267a6SChuck Lever 2055ee0ac0c2SChuck Lever if (!transport->inet) { 2056b0d93ad5SChuck Lever struct sock *sk = sock->sk; 2057b0d93ad5SChuck Lever 2058b0d93ad5SChuck Lever write_lock_bh(&sk->sk_callback_lock); 2059b0d93ad5SChuck Lever 20602a9e1cfaSTrond Myklebust xs_save_old_callbacks(transport, sk); 20612a9e1cfaSTrond Myklebust 2062b0d93ad5SChuck Lever sk->sk_user_data = xprt; 2063f9b2ee71STrond Myklebust sk->sk_data_ready = xs_data_ready; 2064b0d93ad5SChuck Lever sk->sk_write_space = xs_udp_write_space; 2065b4411457SEric Dumazet sock_set_flag(sk, SOCK_FASYNC); 2066b0d93ad5SChuck Lever 2067b0d93ad5SChuck Lever xprt_set_connected(xprt); 2068b0d93ad5SChuck Lever 2069b0d93ad5SChuck Lever /* Reset to new socket */ 2070ee0ac0c2SChuck Lever transport->sock = sock; 2071ee0ac0c2SChuck Lever transport->inet = sk; 2072b0d93ad5SChuck Lever 2073a564b8f0SMel Gorman xs_set_memalloc(xprt); 2074a564b8f0SMel Gorman 2075b0d93ad5SChuck Lever write_unlock_bh(&sk->sk_callback_lock); 2076b0d93ad5SChuck Lever } 2077470056c2SChuck Lever xs_udp_do_set_buffer_size(xprt); 207802910177STrond Myklebust 207902910177STrond Myklebust xprt->stat.connect_start = jiffies; 208016be2d20SChuck Lever } 208116be2d20SChuck Lever 20828c14ff2aSPavel Emelyanov static void xs_udp_setup_socket(struct work_struct *work) 2083a246b010SChuck Lever { 2084a246b010SChuck Lever struct sock_xprt *transport = 2085a246b010SChuck Lever container_of(work, struct sock_xprt, connect_worker.work); 2086a246b010SChuck Lever struct rpc_xprt *xprt = &transport->xprt; 2087d099b8afSColin Ian King struct socket *sock; 2088b65c0310SPavel Emelyanov int status = -EIO; 2089a246b010SChuck Lever 20908c14ff2aSPavel Emelyanov sock = xs_create_sock(xprt, transport, 20914dda9c8aSTrond Myklebust xs_addr(xprt)->sa_family, SOCK_DGRAM, 20924dda9c8aSTrond Myklebust IPPROTO_UDP, false); 2093b65c0310SPavel Emelyanov if (IS_ERR(sock)) 2094a246b010SChuck Lever goto out; 209568e220bdSChuck Lever 2096c740eff8SChuck Lever dprintk("RPC: worker connecting xprt %p via %s to " 2097c740eff8SChuck Lever "%s (port %s)\n", xprt, 2098c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO], 2099c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2100c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT]); 210168e220bdSChuck Lever 210268e220bdSChuck Lever xs_udp_finish_connecting(xprt, sock); 210340b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, 0); 2104a246b010SChuck Lever status = 0; 2105b0d93ad5SChuck Lever out: 2106b0d93ad5SChuck Lever xprt_clear_connecting(xprt); 2107cf76785dSTrond Myklebust xprt_unlock_connect(xprt, transport); 21087d1e8255STrond Myklebust xprt_wake_pending_tasks(xprt, status); 2109b0d93ad5SChuck Lever } 2110b0d93ad5SChuck Lever 21114876cc77STrond Myklebust /** 21124876cc77STrond Myklebust * xs_tcp_shutdown - gracefully shut down a TCP socket 21134876cc77STrond Myklebust * @xprt: transport 21144876cc77STrond Myklebust * 21154876cc77STrond Myklebust * Initiates a graceful shutdown of the TCP socket by calling the 21164876cc77STrond Myklebust * equivalent of shutdown(SHUT_RDWR); 21174876cc77STrond Myklebust */ 21184876cc77STrond Myklebust static void xs_tcp_shutdown(struct rpc_xprt *xprt) 21194876cc77STrond Myklebust { 21204876cc77STrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 21214876cc77STrond Myklebust struct socket *sock = transport->sock; 21229b30889cSTrond Myklebust int skst = transport->inet ? transport->inet->sk_state : TCP_CLOSE; 21234876cc77STrond Myklebust 21244876cc77STrond Myklebust if (sock == NULL) 21254876cc77STrond Myklebust return; 21269b30889cSTrond Myklebust switch (skst) { 21279b30889cSTrond Myklebust default: 21284876cc77STrond Myklebust kernel_sock_shutdown(sock, SHUT_RDWR); 21294876cc77STrond Myklebust trace_rpc_socket_shutdown(xprt, sock); 21309b30889cSTrond Myklebust break; 21319b30889cSTrond Myklebust case TCP_CLOSE: 21329b30889cSTrond Myklebust case TCP_TIME_WAIT: 21334876cc77STrond Myklebust xs_reset_transport(transport); 21344876cc77STrond Myklebust } 21359b30889cSTrond Myklebust } 21364876cc77STrond Myklebust 21378d1b8c62STrond Myklebust static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt, 21388d1b8c62STrond Myklebust struct socket *sock) 2139b0d93ad5SChuck Lever { 214016be2d20SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 21417196dbb0STrond Myklebust unsigned int keepidle; 21427196dbb0STrond Myklebust unsigned int keepcnt; 21437f260e85STrond Myklebust unsigned int opt_on = 1; 2144775f06abSTrond Myklebust unsigned int timeo; 21457f260e85STrond Myklebust 21467196dbb0STrond Myklebust spin_lock_bh(&xprt->transport_lock); 21477196dbb0STrond Myklebust keepidle = DIV_ROUND_UP(xprt->timeout->to_initval, HZ); 21487196dbb0STrond Myklebust keepcnt = xprt->timeout->to_retries + 1; 21497196dbb0STrond Myklebust timeo = jiffies_to_msecs(xprt->timeout->to_initval) * 21507196dbb0STrond Myklebust (xprt->timeout->to_retries + 1); 21517196dbb0STrond Myklebust clear_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state); 21527196dbb0STrond Myklebust spin_unlock_bh(&xprt->transport_lock); 21537f260e85STrond Myklebust 21547f260e85STrond Myklebust /* TCP Keepalive options */ 21557f260e85STrond Myklebust kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, 21567f260e85STrond Myklebust (char *)&opt_on, sizeof(opt_on)); 21577f260e85STrond Myklebust kernel_setsockopt(sock, SOL_TCP, TCP_KEEPIDLE, 21587f260e85STrond Myklebust (char *)&keepidle, sizeof(keepidle)); 21597f260e85STrond Myklebust kernel_setsockopt(sock, SOL_TCP, TCP_KEEPINTVL, 21607f260e85STrond Myklebust (char *)&keepidle, sizeof(keepidle)); 21617f260e85STrond Myklebust kernel_setsockopt(sock, SOL_TCP, TCP_KEEPCNT, 21627f260e85STrond Myklebust (char *)&keepcnt, sizeof(keepcnt)); 2163b0d93ad5SChuck Lever 21648d1b8c62STrond Myklebust /* TCP user timeout (see RFC5482) */ 21658d1b8c62STrond Myklebust kernel_setsockopt(sock, SOL_TCP, TCP_USER_TIMEOUT, 21668d1b8c62STrond Myklebust (char *)&timeo, sizeof(timeo)); 21678d1b8c62STrond Myklebust } 21688d1b8c62STrond Myklebust 21697196dbb0STrond Myklebust static void xs_tcp_set_connect_timeout(struct rpc_xprt *xprt, 21707196dbb0STrond Myklebust unsigned long connect_timeout, 21717196dbb0STrond Myklebust unsigned long reconnect_timeout) 21727196dbb0STrond Myklebust { 21737196dbb0STrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 21747196dbb0STrond Myklebust struct rpc_timeout to; 21757196dbb0STrond Myklebust unsigned long initval; 21767196dbb0STrond Myklebust 21777196dbb0STrond Myklebust spin_lock_bh(&xprt->transport_lock); 21787196dbb0STrond Myklebust if (reconnect_timeout < xprt->max_reconnect_timeout) 21797196dbb0STrond Myklebust xprt->max_reconnect_timeout = reconnect_timeout; 21807196dbb0STrond Myklebust if (connect_timeout < xprt->connect_timeout) { 21817196dbb0STrond Myklebust memcpy(&to, xprt->timeout, sizeof(to)); 21827196dbb0STrond Myklebust initval = DIV_ROUND_UP(connect_timeout, to.to_retries + 1); 21837196dbb0STrond Myklebust /* Arbitrary lower limit */ 21847196dbb0STrond Myklebust if (initval < XS_TCP_INIT_REEST_TO << 1) 21857196dbb0STrond Myklebust initval = XS_TCP_INIT_REEST_TO << 1; 21867196dbb0STrond Myklebust to.to_initval = initval; 21877196dbb0STrond Myklebust to.to_maxval = initval; 21887196dbb0STrond Myklebust memcpy(&transport->tcp_timeout, &to, 21897196dbb0STrond Myklebust sizeof(transport->tcp_timeout)); 21907196dbb0STrond Myklebust xprt->timeout = &transport->tcp_timeout; 21917196dbb0STrond Myklebust xprt->connect_timeout = connect_timeout; 21927196dbb0STrond Myklebust } 21937196dbb0STrond Myklebust set_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state); 21947196dbb0STrond Myklebust spin_unlock_bh(&xprt->transport_lock); 21957196dbb0STrond Myklebust } 21967196dbb0STrond Myklebust 21978d1b8c62STrond Myklebust static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock) 21988d1b8c62STrond Myklebust { 21998d1b8c62STrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 22008d1b8c62STrond Myklebust int ret = -ENOTCONN; 22018d1b8c62STrond Myklebust 22028d1b8c62STrond Myklebust if (!transport->inet) { 22038d1b8c62STrond Myklebust struct sock *sk = sock->sk; 22048d1b8c62STrond Myklebust unsigned int addr_pref = IPV6_PREFER_SRC_PUBLIC; 22058d1b8c62STrond Myklebust 2206d88e4d82SNeilBrown /* Avoid temporary address, they are bad for long-lived 2207d88e4d82SNeilBrown * connections such as NFS mounts. 2208d88e4d82SNeilBrown * RFC4941, section 3.6 suggests that: 2209d88e4d82SNeilBrown * Individual applications, which have specific 2210d88e4d82SNeilBrown * knowledge about the normal duration of connections, 2211d88e4d82SNeilBrown * MAY override this as appropriate. 2212d88e4d82SNeilBrown */ 2213d88e4d82SNeilBrown kernel_setsockopt(sock, SOL_IPV6, IPV6_ADDR_PREFERENCES, 2214d88e4d82SNeilBrown (char *)&addr_pref, sizeof(addr_pref)); 2215d88e4d82SNeilBrown 22168d1b8c62STrond Myklebust xs_tcp_set_socket_timeouts(xprt, sock); 2217775f06abSTrond Myklebust 2218b0d93ad5SChuck Lever write_lock_bh(&sk->sk_callback_lock); 2219b0d93ad5SChuck Lever 22202a9e1cfaSTrond Myklebust xs_save_old_callbacks(transport, sk); 22212a9e1cfaSTrond Myklebust 2222b0d93ad5SChuck Lever sk->sk_user_data = xprt; 22235157b956STrond Myklebust sk->sk_data_ready = xs_data_ready; 2224b0d93ad5SChuck Lever sk->sk_state_change = xs_tcp_state_change; 2225b0d93ad5SChuck Lever sk->sk_write_space = xs_tcp_write_space; 2226b4411457SEric Dumazet sock_set_flag(sk, SOCK_FASYNC); 22272118071dSTrond Myklebust sk->sk_error_report = xs_error_report; 22283167e12cSChuck Lever 22293167e12cSChuck Lever /* socket options */ 22303167e12cSChuck Lever sock_reset_flag(sk, SOCK_LINGER); 22313167e12cSChuck Lever tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF; 2232b0d93ad5SChuck Lever 2233b0d93ad5SChuck Lever xprt_clear_connected(xprt); 2234b0d93ad5SChuck Lever 2235b0d93ad5SChuck Lever /* Reset to new socket */ 2236ee0ac0c2SChuck Lever transport->sock = sock; 2237ee0ac0c2SChuck Lever transport->inet = sk; 2238b0d93ad5SChuck Lever 2239b0d93ad5SChuck Lever write_unlock_bh(&sk->sk_callback_lock); 2240b0d93ad5SChuck Lever } 2241b0d93ad5SChuck Lever 224201d37c42STrond Myklebust if (!xprt_bound(xprt)) 2243fe19a96bSTrond Myklebust goto out; 224401d37c42STrond Myklebust 2245a564b8f0SMel Gorman xs_set_memalloc(xprt); 2246a564b8f0SMel Gorman 2247e1806c7bSTrond Myklebust /* Reset TCP record info */ 2248550aebfeSTrond Myklebust xs_stream_reset_connect(transport); 2249e1806c7bSTrond Myklebust 2250b0d93ad5SChuck Lever /* Tell the socket layer to start connecting... */ 22510fdea1e8STrond Myklebust set_bit(XPRT_SOCK_CONNECTING, &transport->sock_state); 2252fe19a96bSTrond Myklebust ret = kernel_connect(sock, xs_addr(xprt), xprt->addrlen, O_NONBLOCK); 2253fe19a96bSTrond Myklebust switch (ret) { 2254fe19a96bSTrond Myklebust case 0: 22554dda9c8aSTrond Myklebust xs_set_srcport(transport, sock); 2256e9d47639SGustavo A. R. Silva /* fall through */ 2257fe19a96bSTrond Myklebust case -EINPROGRESS: 2258fe19a96bSTrond Myklebust /* SYN_SENT! */ 2259fe19a96bSTrond Myklebust if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO) 2260fe19a96bSTrond Myklebust xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 22611f4c17a0STrond Myklebust break; 22621f4c17a0STrond Myklebust case -EADDRNOTAVAIL: 22631f4c17a0STrond Myklebust /* Source port number is unavailable. Try a new one! */ 22641f4c17a0STrond Myklebust transport->srcport = 0; 2265fe19a96bSTrond Myklebust } 2266fe19a96bSTrond Myklebust out: 2267fe19a96bSTrond Myklebust return ret; 226816be2d20SChuck Lever } 226916be2d20SChuck Lever 227016be2d20SChuck Lever /** 2271b61d59ffSTrond Myklebust * xs_tcp_setup_socket - create a TCP socket and connect to a remote endpoint 2272acf0a39fSChuck Lever * @work: queued work item 227316be2d20SChuck Lever * 227416be2d20SChuck Lever * Invoked by a work queue tasklet. 227516be2d20SChuck Lever */ 2276cdd518d5SPavel Emelyanov static void xs_tcp_setup_socket(struct work_struct *work) 227716be2d20SChuck Lever { 2278cdd518d5SPavel Emelyanov struct sock_xprt *transport = 2279cdd518d5SPavel Emelyanov container_of(work, struct sock_xprt, connect_worker.work); 228016be2d20SChuck Lever struct socket *sock = transport->sock; 2281a9f5f0f7SPavel Emelyanov struct rpc_xprt *xprt = &transport->xprt; 2282b61d59ffSTrond Myklebust int status = -EIO; 228316be2d20SChuck Lever 228416be2d20SChuck Lever if (!sock) { 2285cdd518d5SPavel Emelyanov sock = xs_create_sock(xprt, transport, 22864dda9c8aSTrond Myklebust xs_addr(xprt)->sa_family, SOCK_STREAM, 22874dda9c8aSTrond Myklebust IPPROTO_TCP, true); 2288b61d59ffSTrond Myklebust if (IS_ERR(sock)) { 2289b61d59ffSTrond Myklebust status = PTR_ERR(sock); 229016be2d20SChuck Lever goto out; 229116be2d20SChuck Lever } 22927d1e8255STrond Myklebust } 22937d1e8255STrond Myklebust 2294c740eff8SChuck Lever dprintk("RPC: worker connecting xprt %p via %s to " 2295c740eff8SChuck Lever "%s (port %s)\n", xprt, 2296c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO], 2297c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2298c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT]); 229916be2d20SChuck Lever 230016be2d20SChuck Lever status = xs_tcp_finish_connecting(xprt, sock); 230140b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, status); 2302a246b010SChuck Lever dprintk("RPC: %p connect status %d connected %d sock state %d\n", 230346121cf7SChuck Lever xprt, -status, xprt_connected(xprt), 230446121cf7SChuck Lever sock->sk->sk_state); 2305a246b010SChuck Lever switch (status) { 2306f75e6745STrond Myklebust default: 2307f75e6745STrond Myklebust printk("%s: connect returned unhandled error %d\n", 2308f75e6745STrond Myklebust __func__, status); 2309e9d47639SGustavo A. R. Silva /* fall through */ 2310f75e6745STrond Myklebust case -EADDRNOTAVAIL: 2311f75e6745STrond Myklebust /* We're probably in TIME_WAIT. Get rid of existing socket, 2312f75e6745STrond Myklebust * and retry 2313f75e6745STrond Myklebust */ 2314a519fc7aSTrond Myklebust xs_tcp_force_close(xprt); 231588b5ed73STrond Myklebust break; 23162a491991STrond Myklebust case 0: 2317a246b010SChuck Lever case -EINPROGRESS: 2318a246b010SChuck Lever case -EALREADY: 2319718ba5b8STrond Myklebust xprt_unlock_connect(xprt, transport); 23207d1e8255STrond Myklebust return; 23219fcfe0c8STrond Myklebust case -EINVAL: 23229fcfe0c8STrond Myklebust /* Happens, for instance, if the user specified a link 23239fcfe0c8STrond Myklebust * local IPv6 address without a scope-id. 23249fcfe0c8STrond Myklebust */ 23253ed5e2a2STrond Myklebust case -ECONNREFUSED: 23263ed5e2a2STrond Myklebust case -ECONNRESET: 2327eb5b46faSTrond Myklebust case -ENETDOWN: 23283ed5e2a2STrond Myklebust case -ENETUNREACH: 23294ba161a7STrond Myklebust case -EHOSTUNREACH: 23303913c78cSTrond Myklebust case -EADDRINUSE: 23313601c4a9STrond Myklebust case -ENOBUFS: 23326ea44adcSNeilBrown /* 23336ea44adcSNeilBrown * xs_tcp_force_close() wakes tasks with -EIO. 23346ea44adcSNeilBrown * We need to wake them first to ensure the 23356ea44adcSNeilBrown * correct error code. 23366ea44adcSNeilBrown */ 23376ea44adcSNeilBrown xprt_wake_pending_tasks(xprt, status); 23384efdd92cSTrond Myklebust xs_tcp_force_close(xprt); 23399fcfe0c8STrond Myklebust goto out; 23408a2cec29STrond Myklebust } 23412a491991STrond Myklebust status = -EAGAIN; 2342a246b010SChuck Lever out: 23432226feb6SChuck Lever xprt_clear_connecting(xprt); 2344cf76785dSTrond Myklebust xprt_unlock_connect(xprt, transport); 23457d1e8255STrond Myklebust xprt_wake_pending_tasks(xprt, status); 2346a246b010SChuck Lever } 2347a246b010SChuck Lever 234802910177STrond Myklebust static unsigned long xs_reconnect_delay(const struct rpc_xprt *xprt) 234902910177STrond Myklebust { 235002910177STrond Myklebust unsigned long start, now = jiffies; 235102910177STrond Myklebust 235202910177STrond Myklebust start = xprt->stat.connect_start + xprt->reestablish_timeout; 235302910177STrond Myklebust if (time_after(start, now)) 235402910177STrond Myklebust return start - now; 235502910177STrond Myklebust return 0; 235602910177STrond Myklebust } 235702910177STrond Myklebust 23583851f1cdSTrond Myklebust static void xs_reconnect_backoff(struct rpc_xprt *xprt) 23593851f1cdSTrond Myklebust { 23603851f1cdSTrond Myklebust xprt->reestablish_timeout <<= 1; 23613851f1cdSTrond Myklebust if (xprt->reestablish_timeout > xprt->max_reconnect_timeout) 23623851f1cdSTrond Myklebust xprt->reestablish_timeout = xprt->max_reconnect_timeout; 23633851f1cdSTrond Myklebust if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO) 23643851f1cdSTrond Myklebust xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 23653851f1cdSTrond Myklebust } 23663851f1cdSTrond Myklebust 236768e220bdSChuck Lever /** 23689903cd1cSChuck Lever * xs_connect - connect a socket to a remote endpoint 23691b092092STrond Myklebust * @xprt: pointer to transport structure 23709903cd1cSChuck Lever * @task: address of RPC task that manages state of connect request 23719903cd1cSChuck Lever * 23729903cd1cSChuck Lever * TCP: If the remote end dropped the connection, delay reconnecting. 237303bf4b70SChuck Lever * 237403bf4b70SChuck Lever * UDP socket connects are synchronous, but we use a work queue anyway 237503bf4b70SChuck Lever * to guarantee that even unprivileged user processes can set up a 237603bf4b70SChuck Lever * socket on a privileged port. 237703bf4b70SChuck Lever * 237803bf4b70SChuck Lever * If a UDP socket connect fails, the delay behavior here prevents 237903bf4b70SChuck Lever * retry floods (hard mounts). 23809903cd1cSChuck Lever */ 23811b092092STrond Myklebust static void xs_connect(struct rpc_xprt *xprt, struct rpc_task *task) 2382a246b010SChuck Lever { 2383ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 238402910177STrond Myklebust unsigned long delay = 0; 2385a246b010SChuck Lever 2386718ba5b8STrond Myklebust WARN_ON_ONCE(!xprt_lock_connect(xprt, task, transport)); 2387718ba5b8STrond Myklebust 238899b1a4c3STrond Myklebust if (transport->sock != NULL) { 238946121cf7SChuck Lever dprintk("RPC: xs_connect delayed xprt %p for %lu " 239046121cf7SChuck Lever "seconds\n", 239103bf4b70SChuck Lever xprt, xprt->reestablish_timeout / HZ); 239299b1a4c3STrond Myklebust 239399b1a4c3STrond Myklebust /* Start by resetting any existing state */ 239499b1a4c3STrond Myklebust xs_reset_transport(transport); 239599b1a4c3STrond Myklebust 239602910177STrond Myklebust delay = xs_reconnect_delay(xprt); 23973851f1cdSTrond Myklebust xs_reconnect_backoff(xprt); 239802910177STrond Myklebust 239902910177STrond Myklebust } else 24009903cd1cSChuck Lever dprintk("RPC: xs_connect scheduled xprt %p\n", xprt); 240102910177STrond Myklebust 240240a5f1b1STrond Myklebust queue_delayed_work(xprtiod_workqueue, 240302910177STrond Myklebust &transport->connect_worker, 240402910177STrond Myklebust delay); 2405a246b010SChuck Lever } 2406a246b010SChuck Lever 2407262ca07dSChuck Lever /** 2408176e21eeSChuck Lever * xs_local_print_stats - display AF_LOCAL socket-specifc stats 2409176e21eeSChuck Lever * @xprt: rpc_xprt struct containing statistics 2410176e21eeSChuck Lever * @seq: output file 2411176e21eeSChuck Lever * 2412176e21eeSChuck Lever */ 2413176e21eeSChuck Lever static void xs_local_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) 2414176e21eeSChuck Lever { 2415176e21eeSChuck Lever long idle_time = 0; 2416176e21eeSChuck Lever 2417176e21eeSChuck Lever if (xprt_connected(xprt)) 2418176e21eeSChuck Lever idle_time = (long)(jiffies - xprt->last_used) / HZ; 2419176e21eeSChuck Lever 2420176e21eeSChuck Lever seq_printf(seq, "\txprt:\tlocal %lu %lu %lu %ld %lu %lu %lu " 242115a45206SAndy Adamson "%llu %llu %lu %llu %llu\n", 2422176e21eeSChuck Lever xprt->stat.bind_count, 2423176e21eeSChuck Lever xprt->stat.connect_count, 24248440a886SChuck Lever xprt->stat.connect_time / HZ, 2425176e21eeSChuck Lever idle_time, 2426176e21eeSChuck Lever xprt->stat.sends, 2427176e21eeSChuck Lever xprt->stat.recvs, 2428176e21eeSChuck Lever xprt->stat.bad_xids, 2429176e21eeSChuck Lever xprt->stat.req_u, 243015a45206SAndy Adamson xprt->stat.bklog_u, 243115a45206SAndy Adamson xprt->stat.max_slots, 243215a45206SAndy Adamson xprt->stat.sending_u, 243315a45206SAndy Adamson xprt->stat.pending_u); 2434176e21eeSChuck Lever } 2435176e21eeSChuck Lever 2436176e21eeSChuck Lever /** 2437262ca07dSChuck Lever * xs_udp_print_stats - display UDP socket-specifc stats 2438262ca07dSChuck Lever * @xprt: rpc_xprt struct containing statistics 2439262ca07dSChuck Lever * @seq: output file 2440262ca07dSChuck Lever * 2441262ca07dSChuck Lever */ 2442262ca07dSChuck Lever static void xs_udp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) 2443262ca07dSChuck Lever { 2444c8475461SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2445c8475461SChuck Lever 244615a45206SAndy Adamson seq_printf(seq, "\txprt:\tudp %u %lu %lu %lu %lu %llu %llu " 244715a45206SAndy Adamson "%lu %llu %llu\n", 2448fbfffbd5SChuck Lever transport->srcport, 2449262ca07dSChuck Lever xprt->stat.bind_count, 2450262ca07dSChuck Lever xprt->stat.sends, 2451262ca07dSChuck Lever xprt->stat.recvs, 2452262ca07dSChuck Lever xprt->stat.bad_xids, 2453262ca07dSChuck Lever xprt->stat.req_u, 245415a45206SAndy Adamson xprt->stat.bklog_u, 245515a45206SAndy Adamson xprt->stat.max_slots, 245615a45206SAndy Adamson xprt->stat.sending_u, 245715a45206SAndy Adamson xprt->stat.pending_u); 2458262ca07dSChuck Lever } 2459262ca07dSChuck Lever 2460262ca07dSChuck Lever /** 2461262ca07dSChuck Lever * xs_tcp_print_stats - display TCP socket-specifc stats 2462262ca07dSChuck Lever * @xprt: rpc_xprt struct containing statistics 2463262ca07dSChuck Lever * @seq: output file 2464262ca07dSChuck Lever * 2465262ca07dSChuck Lever */ 2466262ca07dSChuck Lever static void xs_tcp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) 2467262ca07dSChuck Lever { 2468c8475461SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2469262ca07dSChuck Lever long idle_time = 0; 2470262ca07dSChuck Lever 2471262ca07dSChuck Lever if (xprt_connected(xprt)) 2472262ca07dSChuck Lever idle_time = (long)(jiffies - xprt->last_used) / HZ; 2473262ca07dSChuck Lever 247415a45206SAndy Adamson seq_printf(seq, "\txprt:\ttcp %u %lu %lu %lu %ld %lu %lu %lu " 247515a45206SAndy Adamson "%llu %llu %lu %llu %llu\n", 2476fbfffbd5SChuck Lever transport->srcport, 2477262ca07dSChuck Lever xprt->stat.bind_count, 2478262ca07dSChuck Lever xprt->stat.connect_count, 24798440a886SChuck Lever xprt->stat.connect_time / HZ, 2480262ca07dSChuck Lever idle_time, 2481262ca07dSChuck Lever xprt->stat.sends, 2482262ca07dSChuck Lever xprt->stat.recvs, 2483262ca07dSChuck Lever xprt->stat.bad_xids, 2484262ca07dSChuck Lever xprt->stat.req_u, 248515a45206SAndy Adamson xprt->stat.bklog_u, 248615a45206SAndy Adamson xprt->stat.max_slots, 248715a45206SAndy Adamson xprt->stat.sending_u, 248815a45206SAndy Adamson xprt->stat.pending_u); 2489262ca07dSChuck Lever } 2490262ca07dSChuck Lever 24914cfc7e60SRahul Iyer /* 24924cfc7e60SRahul Iyer * Allocate a bunch of pages for a scratch buffer for the rpc code. The reason 24934cfc7e60SRahul Iyer * we allocate pages instead doing a kmalloc like rpc_malloc is because we want 24944cfc7e60SRahul Iyer * to use the server side send routines. 24954cfc7e60SRahul Iyer */ 24965fe6eaa1SChuck Lever static int bc_malloc(struct rpc_task *task) 24974cfc7e60SRahul Iyer { 24985fe6eaa1SChuck Lever struct rpc_rqst *rqst = task->tk_rqstp; 24995fe6eaa1SChuck Lever size_t size = rqst->rq_callsize; 25004cfc7e60SRahul Iyer struct page *page; 25014cfc7e60SRahul Iyer struct rpc_buffer *buf; 25024cfc7e60SRahul Iyer 25035fe6eaa1SChuck Lever if (size > PAGE_SIZE - sizeof(struct rpc_buffer)) { 25045fe6eaa1SChuck Lever WARN_ONCE(1, "xprtsock: large bc buffer request (size %zu)\n", 25055fe6eaa1SChuck Lever size); 25065fe6eaa1SChuck Lever return -EINVAL; 25075fe6eaa1SChuck Lever } 25084cfc7e60SRahul Iyer 2509b8a13d03SWeston Andros Adamson page = alloc_page(GFP_KERNEL); 25104cfc7e60SRahul Iyer if (!page) 25115fe6eaa1SChuck Lever return -ENOMEM; 25124cfc7e60SRahul Iyer 25134cfc7e60SRahul Iyer buf = page_address(page); 25144cfc7e60SRahul Iyer buf->len = PAGE_SIZE; 25154cfc7e60SRahul Iyer 25165fe6eaa1SChuck Lever rqst->rq_buffer = buf->data; 251718e601d6SJeff Layton rqst->rq_rbuffer = (char *)rqst->rq_buffer + rqst->rq_callsize; 25185fe6eaa1SChuck Lever return 0; 25194cfc7e60SRahul Iyer } 25204cfc7e60SRahul Iyer 25214cfc7e60SRahul Iyer /* 25224cfc7e60SRahul Iyer * Free the space allocated in the bc_alloc routine 25234cfc7e60SRahul Iyer */ 25243435c74aSChuck Lever static void bc_free(struct rpc_task *task) 25254cfc7e60SRahul Iyer { 25263435c74aSChuck Lever void *buffer = task->tk_rqstp->rq_buffer; 25274cfc7e60SRahul Iyer struct rpc_buffer *buf; 25284cfc7e60SRahul Iyer 25294cfc7e60SRahul Iyer buf = container_of(buffer, struct rpc_buffer, data); 25304cfc7e60SRahul Iyer free_page((unsigned long)buf); 25314cfc7e60SRahul Iyer } 25324cfc7e60SRahul Iyer 25334cfc7e60SRahul Iyer /* 25344cfc7e60SRahul Iyer * Use the svc_sock to send the callback. Must be called with svsk->sk_mutex 25354cfc7e60SRahul Iyer * held. Borrows heavily from svc_tcp_sendto and xs_tcp_send_request. 25364cfc7e60SRahul Iyer */ 25374cfc7e60SRahul Iyer static int bc_sendto(struct rpc_rqst *req) 25384cfc7e60SRahul Iyer { 25394cfc7e60SRahul Iyer int len; 25404cfc7e60SRahul Iyer struct xdr_buf *xbufp = &req->rq_snd_buf; 25414cfc7e60SRahul Iyer struct rpc_xprt *xprt = req->rq_xprt; 25424cfc7e60SRahul Iyer struct sock_xprt *transport = 25434cfc7e60SRahul Iyer container_of(xprt, struct sock_xprt, xprt); 25444cfc7e60SRahul Iyer struct socket *sock = transport->sock; 25454cfc7e60SRahul Iyer unsigned long headoff; 25464cfc7e60SRahul Iyer unsigned long tailoff; 25474cfc7e60SRahul Iyer 254861677eeeSChuck Lever xs_encode_stream_record_marker(xbufp); 25494cfc7e60SRahul Iyer 25504cfc7e60SRahul Iyer tailoff = (unsigned long)xbufp->tail[0].iov_base & ~PAGE_MASK; 25514cfc7e60SRahul Iyer headoff = (unsigned long)xbufp->head[0].iov_base & ~PAGE_MASK; 25524cfc7e60SRahul Iyer len = svc_send_common(sock, xbufp, 25534cfc7e60SRahul Iyer virt_to_page(xbufp->head[0].iov_base), headoff, 25544cfc7e60SRahul Iyer xbufp->tail[0].iov_base, tailoff); 25554cfc7e60SRahul Iyer 25564cfc7e60SRahul Iyer if (len != xbufp->len) { 25574cfc7e60SRahul Iyer printk(KERN_NOTICE "Error sending entire callback!\n"); 25584cfc7e60SRahul Iyer len = -EAGAIN; 25594cfc7e60SRahul Iyer } 25604cfc7e60SRahul Iyer 25614cfc7e60SRahul Iyer return len; 25624cfc7e60SRahul Iyer } 25634cfc7e60SRahul Iyer 25644cfc7e60SRahul Iyer /* 25654cfc7e60SRahul Iyer * The send routine. Borrows from svc_send 25664cfc7e60SRahul Iyer */ 2567adfa7144STrond Myklebust static int bc_send_request(struct rpc_rqst *req) 25684cfc7e60SRahul Iyer { 25694cfc7e60SRahul Iyer struct svc_xprt *xprt; 25707fc56136SAndrzej Hajda int len; 25714cfc7e60SRahul Iyer 25724cfc7e60SRahul Iyer dprintk("sending request with xid: %08x\n", ntohl(req->rq_xid)); 25734cfc7e60SRahul Iyer /* 25744cfc7e60SRahul Iyer * Get the server socket associated with this callback xprt 25754cfc7e60SRahul Iyer */ 25764cfc7e60SRahul Iyer xprt = req->rq_xprt->bc_xprt; 25774cfc7e60SRahul Iyer 25784cfc7e60SRahul Iyer /* 25794cfc7e60SRahul Iyer * Grab the mutex to serialize data as the connection is shared 25804cfc7e60SRahul Iyer * with the fore channel 25814cfc7e60SRahul Iyer */ 2582c544577dSTrond Myklebust mutex_lock(&xprt->xpt_mutex); 25834cfc7e60SRahul Iyer if (test_bit(XPT_DEAD, &xprt->xpt_flags)) 25844cfc7e60SRahul Iyer len = -ENOTCONN; 25854cfc7e60SRahul Iyer else 25864cfc7e60SRahul Iyer len = bc_sendto(req); 25874cfc7e60SRahul Iyer mutex_unlock(&xprt->xpt_mutex); 25884cfc7e60SRahul Iyer 25894cfc7e60SRahul Iyer if (len > 0) 25904cfc7e60SRahul Iyer len = 0; 25914cfc7e60SRahul Iyer 25924cfc7e60SRahul Iyer return len; 25934cfc7e60SRahul Iyer } 25944cfc7e60SRahul Iyer 25954cfc7e60SRahul Iyer /* 25964cfc7e60SRahul Iyer * The close routine. Since this is client initiated, we do nothing 25974cfc7e60SRahul Iyer */ 25984cfc7e60SRahul Iyer 25994cfc7e60SRahul Iyer static void bc_close(struct rpc_xprt *xprt) 26004cfc7e60SRahul Iyer { 26014cfc7e60SRahul Iyer } 26024cfc7e60SRahul Iyer 26034cfc7e60SRahul Iyer /* 26044cfc7e60SRahul Iyer * The xprt destroy routine. Again, because this connection is client 26054cfc7e60SRahul Iyer * initiated, we do nothing 26064cfc7e60SRahul Iyer */ 26074cfc7e60SRahul Iyer 26084cfc7e60SRahul Iyer static void bc_destroy(struct rpc_xprt *xprt) 26094cfc7e60SRahul Iyer { 261047f72efaSKinglong Mee dprintk("RPC: bc_destroy xprt %p\n", xprt); 261147f72efaSKinglong Mee 261247f72efaSKinglong Mee xs_xprt_free(xprt); 261347f72efaSKinglong Mee module_put(THIS_MODULE); 26144cfc7e60SRahul Iyer } 26154cfc7e60SRahul Iyer 2616d31ae254SChuck Lever static const struct rpc_xprt_ops xs_local_ops = { 2617176e21eeSChuck Lever .reserve_xprt = xprt_reserve_xprt, 26184cd34e7cSTrond Myklebust .release_xprt = xprt_release_xprt, 2619f39c1bfbSTrond Myklebust .alloc_slot = xprt_alloc_slot, 2620a9cde23aSChuck Lever .free_slot = xprt_free_slot, 2621176e21eeSChuck Lever .rpcbind = xs_local_rpcbind, 2622176e21eeSChuck Lever .set_port = xs_local_set_port, 2623dc107402SJ. Bruce Fields .connect = xs_local_connect, 2624176e21eeSChuck Lever .buf_alloc = rpc_malloc, 2625176e21eeSChuck Lever .buf_free = rpc_free, 2626550aebfeSTrond Myklebust .prepare_request = xs_stream_prepare_request, 2627176e21eeSChuck Lever .send_request = xs_local_send_request, 2628176e21eeSChuck Lever .set_retrans_timeout = xprt_set_retrans_timeout_def, 2629176e21eeSChuck Lever .close = xs_close, 2630a1311d87STrond Myklebust .destroy = xs_destroy, 2631176e21eeSChuck Lever .print_stats = xs_local_print_stats, 2632d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2633d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 2634176e21eeSChuck Lever }; 2635176e21eeSChuck Lever 2636d31ae254SChuck Lever static const struct rpc_xprt_ops xs_udp_ops = { 263743118c29SChuck Lever .set_buffer_size = xs_udp_set_buffer_size, 263812a80469SChuck Lever .reserve_xprt = xprt_reserve_xprt_cong, 263949e9a890SChuck Lever .release_xprt = xprt_release_xprt_cong, 2640f39c1bfbSTrond Myklebust .alloc_slot = xprt_alloc_slot, 2641a9cde23aSChuck Lever .free_slot = xprt_free_slot, 264245160d62SChuck Lever .rpcbind = rpcb_getport_async, 264392200412SChuck Lever .set_port = xs_set_port, 26449903cd1cSChuck Lever .connect = xs_connect, 264502107148SChuck Lever .buf_alloc = rpc_malloc, 264602107148SChuck Lever .buf_free = rpc_free, 2647262965f5SChuck Lever .send_request = xs_udp_send_request, 2648fe3aca29SChuck Lever .set_retrans_timeout = xprt_set_retrans_timeout_rtt, 264946c0ee8bSChuck Lever .timer = xs_udp_timer, 2650a58dd398SChuck Lever .release_request = xprt_release_rqst_cong, 2651262965f5SChuck Lever .close = xs_close, 2652262965f5SChuck Lever .destroy = xs_destroy, 2653262ca07dSChuck Lever .print_stats = xs_udp_print_stats, 2654d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2655d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 26564a068258SChuck Lever .inject_disconnect = xs_inject_disconnect, 2657262965f5SChuck Lever }; 2658262965f5SChuck Lever 2659d31ae254SChuck Lever static const struct rpc_xprt_ops xs_tcp_ops = { 266012a80469SChuck Lever .reserve_xprt = xprt_reserve_xprt, 26614cd34e7cSTrond Myklebust .release_xprt = xprt_release_xprt, 266236bd7de9STrond Myklebust .alloc_slot = xprt_alloc_slot, 2663a9cde23aSChuck Lever .free_slot = xprt_free_slot, 266445160d62SChuck Lever .rpcbind = rpcb_getport_async, 266592200412SChuck Lever .set_port = xs_set_port, 26660b9e7943STrond Myklebust .connect = xs_connect, 266702107148SChuck Lever .buf_alloc = rpc_malloc, 266802107148SChuck Lever .buf_free = rpc_free, 2669277e4ab7STrond Myklebust .prepare_request = xs_stream_prepare_request, 2670262965f5SChuck Lever .send_request = xs_tcp_send_request, 2671fe3aca29SChuck Lever .set_retrans_timeout = xprt_set_retrans_timeout_def, 2672c627d31bSTrond Myklebust .close = xs_tcp_shutdown, 26739903cd1cSChuck Lever .destroy = xs_destroy, 26747196dbb0STrond Myklebust .set_connect_timeout = xs_tcp_set_connect_timeout, 2675262ca07dSChuck Lever .print_stats = xs_tcp_print_stats, 2676d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2677d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 26784a068258SChuck Lever .inject_disconnect = xs_inject_disconnect, 267942e5c3e2SChuck Lever #ifdef CONFIG_SUNRPC_BACKCHANNEL 268042e5c3e2SChuck Lever .bc_setup = xprt_setup_bc, 26816b26cc8cSChuck Lever .bc_maxpayload = xs_tcp_bc_maxpayload, 268242e5c3e2SChuck Lever .bc_free_rqst = xprt_free_bc_rqst, 268342e5c3e2SChuck Lever .bc_destroy = xprt_destroy_bc, 268442e5c3e2SChuck Lever #endif 2685a246b010SChuck Lever }; 2686a246b010SChuck Lever 26874cfc7e60SRahul Iyer /* 26884cfc7e60SRahul Iyer * The rpc_xprt_ops for the server backchannel 26894cfc7e60SRahul Iyer */ 26904cfc7e60SRahul Iyer 2691d31ae254SChuck Lever static const struct rpc_xprt_ops bc_tcp_ops = { 26924cfc7e60SRahul Iyer .reserve_xprt = xprt_reserve_xprt, 26934cfc7e60SRahul Iyer .release_xprt = xprt_release_xprt, 269484e28a30SBryan Schumaker .alloc_slot = xprt_alloc_slot, 2695a9cde23aSChuck Lever .free_slot = xprt_free_slot, 26964cfc7e60SRahul Iyer .buf_alloc = bc_malloc, 26974cfc7e60SRahul Iyer .buf_free = bc_free, 26984cfc7e60SRahul Iyer .send_request = bc_send_request, 26994cfc7e60SRahul Iyer .set_retrans_timeout = xprt_set_retrans_timeout_def, 27004cfc7e60SRahul Iyer .close = bc_close, 27014cfc7e60SRahul Iyer .destroy = bc_destroy, 27024cfc7e60SRahul Iyer .print_stats = xs_tcp_print_stats, 2703d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2704d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 27054a068258SChuck Lever .inject_disconnect = xs_inject_disconnect, 27064cfc7e60SRahul Iyer }; 27074cfc7e60SRahul Iyer 270892476850SChuck Lever static int xs_init_anyaddr(const int family, struct sockaddr *sap) 270992476850SChuck Lever { 271092476850SChuck Lever static const struct sockaddr_in sin = { 271192476850SChuck Lever .sin_family = AF_INET, 271292476850SChuck Lever .sin_addr.s_addr = htonl(INADDR_ANY), 271392476850SChuck Lever }; 271492476850SChuck Lever static const struct sockaddr_in6 sin6 = { 271592476850SChuck Lever .sin6_family = AF_INET6, 271692476850SChuck Lever .sin6_addr = IN6ADDR_ANY_INIT, 271792476850SChuck Lever }; 271892476850SChuck Lever 271992476850SChuck Lever switch (family) { 2720176e21eeSChuck Lever case AF_LOCAL: 2721176e21eeSChuck Lever break; 272292476850SChuck Lever case AF_INET: 272392476850SChuck Lever memcpy(sap, &sin, sizeof(sin)); 272492476850SChuck Lever break; 272592476850SChuck Lever case AF_INET6: 272692476850SChuck Lever memcpy(sap, &sin6, sizeof(sin6)); 272792476850SChuck Lever break; 272892476850SChuck Lever default: 272992476850SChuck Lever dprintk("RPC: %s: Bad address family\n", __func__); 273092476850SChuck Lever return -EAFNOSUPPORT; 273192476850SChuck Lever } 273292476850SChuck Lever return 0; 273392476850SChuck Lever } 273492476850SChuck Lever 27353c341b0bS\"Talpey, Thomas\ static struct rpc_xprt *xs_setup_xprt(struct xprt_create *args, 2736d9ba131dSTrond Myklebust unsigned int slot_table_size, 2737d9ba131dSTrond Myklebust unsigned int max_slot_table_size) 2738c8541ecdSChuck Lever { 2739c8541ecdSChuck Lever struct rpc_xprt *xprt; 2740ffc2e518SChuck Lever struct sock_xprt *new; 2741c8541ecdSChuck Lever 274296802a09SFrank van Maarseveen if (args->addrlen > sizeof(xprt->addr)) { 2743c8541ecdSChuck Lever dprintk("RPC: xs_setup_xprt: address too large\n"); 2744c8541ecdSChuck Lever return ERR_PTR(-EBADF); 2745c8541ecdSChuck Lever } 2746c8541ecdSChuck Lever 2747d9ba131dSTrond Myklebust xprt = xprt_alloc(args->net, sizeof(*new), slot_table_size, 2748d9ba131dSTrond Myklebust max_slot_table_size); 2749bd1722d4SPavel Emelyanov if (xprt == NULL) { 275046121cf7SChuck Lever dprintk("RPC: xs_setup_xprt: couldn't allocate " 275146121cf7SChuck Lever "rpc_xprt\n"); 2752c8541ecdSChuck Lever return ERR_PTR(-ENOMEM); 2753c8541ecdSChuck Lever } 2754c8541ecdSChuck Lever 2755bd1722d4SPavel Emelyanov new = container_of(xprt, struct sock_xprt, xprt); 2756edc1b01cSTrond Myklebust mutex_init(&new->recv_mutex); 275796802a09SFrank van Maarseveen memcpy(&xprt->addr, args->dstaddr, args->addrlen); 275896802a09SFrank van Maarseveen xprt->addrlen = args->addrlen; 2759d3bc9a1dSFrank van Maarseveen if (args->srcaddr) 2760fbfffbd5SChuck Lever memcpy(&new->srcaddr, args->srcaddr, args->addrlen); 276192476850SChuck Lever else { 276292476850SChuck Lever int err; 276392476850SChuck Lever err = xs_init_anyaddr(args->dstaddr->sa_family, 276492476850SChuck Lever (struct sockaddr *)&new->srcaddr); 27652aa13531SStanislav Kinsbursky if (err != 0) { 27662aa13531SStanislav Kinsbursky xprt_free(xprt); 276792476850SChuck Lever return ERR_PTR(err); 276892476850SChuck Lever } 27692aa13531SStanislav Kinsbursky } 2770c8541ecdSChuck Lever 2771c8541ecdSChuck Lever return xprt; 2772c8541ecdSChuck Lever } 2773c8541ecdSChuck Lever 2774176e21eeSChuck Lever static const struct rpc_timeout xs_local_default_timeout = { 2775176e21eeSChuck Lever .to_initval = 10 * HZ, 2776176e21eeSChuck Lever .to_maxval = 10 * HZ, 2777176e21eeSChuck Lever .to_retries = 2, 2778176e21eeSChuck Lever }; 2779176e21eeSChuck Lever 2780176e21eeSChuck Lever /** 2781176e21eeSChuck Lever * xs_setup_local - Set up transport to use an AF_LOCAL socket 2782176e21eeSChuck Lever * @args: rpc transport creation arguments 2783176e21eeSChuck Lever * 2784176e21eeSChuck Lever * AF_LOCAL is a "tpi_cots_ord" transport, just like TCP 2785176e21eeSChuck Lever */ 2786176e21eeSChuck Lever static struct rpc_xprt *xs_setup_local(struct xprt_create *args) 2787176e21eeSChuck Lever { 2788176e21eeSChuck Lever struct sockaddr_un *sun = (struct sockaddr_un *)args->dstaddr; 2789176e21eeSChuck Lever struct sock_xprt *transport; 2790176e21eeSChuck Lever struct rpc_xprt *xprt; 2791176e21eeSChuck Lever struct rpc_xprt *ret; 2792176e21eeSChuck Lever 2793d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries, 2794d9ba131dSTrond Myklebust xprt_max_tcp_slot_table_entries); 2795176e21eeSChuck Lever if (IS_ERR(xprt)) 2796176e21eeSChuck Lever return xprt; 2797176e21eeSChuck Lever transport = container_of(xprt, struct sock_xprt, xprt); 2798176e21eeSChuck Lever 2799176e21eeSChuck Lever xprt->prot = 0; 2800176e21eeSChuck Lever xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32); 2801176e21eeSChuck Lever xprt->max_payload = RPC_MAX_FRAGMENT_SIZE; 2802176e21eeSChuck Lever 2803176e21eeSChuck Lever xprt->bind_timeout = XS_BIND_TO; 2804176e21eeSChuck Lever xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 2805176e21eeSChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO; 2806176e21eeSChuck Lever 2807176e21eeSChuck Lever xprt->ops = &xs_local_ops; 2808176e21eeSChuck Lever xprt->timeout = &xs_local_default_timeout; 2809176e21eeSChuck Lever 2810550aebfeSTrond Myklebust INIT_WORK(&transport->recv_worker, xs_stream_data_receive_workfn); 2811550aebfeSTrond Myklebust INIT_DELAYED_WORK(&transport->connect_worker, xs_dummy_setup_socket); 281293dc41bdSNeilBrown 2813176e21eeSChuck Lever switch (sun->sun_family) { 2814176e21eeSChuck Lever case AF_LOCAL: 2815176e21eeSChuck Lever if (sun->sun_path[0] != '/') { 2816176e21eeSChuck Lever dprintk("RPC: bad AF_LOCAL address: %s\n", 2817176e21eeSChuck Lever sun->sun_path); 2818176e21eeSChuck Lever ret = ERR_PTR(-EINVAL); 2819176e21eeSChuck Lever goto out_err; 2820176e21eeSChuck Lever } 2821176e21eeSChuck Lever xprt_set_bound(xprt); 2822176e21eeSChuck Lever xs_format_peer_addresses(xprt, "local", RPCBIND_NETID_LOCAL); 28237073ea87SJ. Bruce Fields ret = ERR_PTR(xs_local_setup_socket(transport)); 28247073ea87SJ. Bruce Fields if (ret) 28257073ea87SJ. Bruce Fields goto out_err; 2826176e21eeSChuck Lever break; 2827176e21eeSChuck Lever default: 2828176e21eeSChuck Lever ret = ERR_PTR(-EAFNOSUPPORT); 2829176e21eeSChuck Lever goto out_err; 2830176e21eeSChuck Lever } 2831176e21eeSChuck Lever 2832176e21eeSChuck Lever dprintk("RPC: set up xprt to %s via AF_LOCAL\n", 2833176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR]); 2834176e21eeSChuck Lever 2835176e21eeSChuck Lever if (try_module_get(THIS_MODULE)) 2836176e21eeSChuck Lever return xprt; 2837176e21eeSChuck Lever ret = ERR_PTR(-EINVAL); 2838176e21eeSChuck Lever out_err: 2839315f3812SKinglong Mee xs_xprt_free(xprt); 2840176e21eeSChuck Lever return ret; 2841176e21eeSChuck Lever } 2842176e21eeSChuck Lever 28432881ae74STrond Myklebust static const struct rpc_timeout xs_udp_default_timeout = { 28442881ae74STrond Myklebust .to_initval = 5 * HZ, 28452881ae74STrond Myklebust .to_maxval = 30 * HZ, 28462881ae74STrond Myklebust .to_increment = 5 * HZ, 28472881ae74STrond Myklebust .to_retries = 5, 28482881ae74STrond Myklebust }; 28492881ae74STrond Myklebust 28509903cd1cSChuck Lever /** 28519903cd1cSChuck Lever * xs_setup_udp - Set up transport to use a UDP socket 285296802a09SFrank van Maarseveen * @args: rpc transport creation arguments 28539903cd1cSChuck Lever * 28549903cd1cSChuck Lever */ 2855483066d6SAdrian Bunk static struct rpc_xprt *xs_setup_udp(struct xprt_create *args) 2856a246b010SChuck Lever { 28578f9d5b1aSChuck Lever struct sockaddr *addr = args->dstaddr; 2858c8541ecdSChuck Lever struct rpc_xprt *xprt; 2859c8475461SChuck Lever struct sock_xprt *transport; 28600a68b0beSJ. Bruce Fields struct rpc_xprt *ret; 2861a246b010SChuck Lever 2862d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_udp_slot_table_entries, 2863d9ba131dSTrond Myklebust xprt_udp_slot_table_entries); 2864c8541ecdSChuck Lever if (IS_ERR(xprt)) 2865c8541ecdSChuck Lever return xprt; 2866c8475461SChuck Lever transport = container_of(xprt, struct sock_xprt, xprt); 2867a246b010SChuck Lever 2868ec739ef0SChuck Lever xprt->prot = IPPROTO_UDP; 2869808012fbSChuck Lever xprt->tsh_size = 0; 2870a246b010SChuck Lever /* XXX: header size can vary due to auth type, IPv6, etc. */ 2871a246b010SChuck Lever xprt->max_payload = (1U << 16) - (MAX_HEADER << 3); 2872a246b010SChuck Lever 287303bf4b70SChuck Lever xprt->bind_timeout = XS_BIND_TO; 287403bf4b70SChuck Lever xprt->reestablish_timeout = XS_UDP_REEST_TO; 287503bf4b70SChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO; 2876a246b010SChuck Lever 2877262965f5SChuck Lever xprt->ops = &xs_udp_ops; 2878a246b010SChuck Lever 2879ba7392bbSTrond Myklebust xprt->timeout = &xs_udp_default_timeout; 2880a246b010SChuck Lever 2881f9b2ee71STrond Myklebust INIT_WORK(&transport->recv_worker, xs_udp_data_receive_workfn); 2882edc1b01cSTrond Myklebust INIT_DELAYED_WORK(&transport->connect_worker, xs_udp_setup_socket); 2883edc1b01cSTrond Myklebust 28848f9d5b1aSChuck Lever switch (addr->sa_family) { 28858f9d5b1aSChuck Lever case AF_INET: 28868f9d5b1aSChuck Lever if (((struct sockaddr_in *)addr)->sin_port != htons(0)) 28878f9d5b1aSChuck Lever xprt_set_bound(xprt); 28888f9d5b1aSChuck Lever 28899dc3b095SChuck Lever xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP); 28908f9d5b1aSChuck Lever break; 28918f9d5b1aSChuck Lever case AF_INET6: 28928f9d5b1aSChuck Lever if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0)) 28938f9d5b1aSChuck Lever xprt_set_bound(xprt); 28948f9d5b1aSChuck Lever 28959dc3b095SChuck Lever xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP6); 28968f9d5b1aSChuck Lever break; 28978f9d5b1aSChuck Lever default: 28980a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT); 28990a68b0beSJ. Bruce Fields goto out_err; 29008f9d5b1aSChuck Lever } 29018f9d5b1aSChuck Lever 2902c740eff8SChuck Lever if (xprt_bound(xprt)) 2903c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (port %s) via %s\n", 2904c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2905c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT], 2906c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 2907c740eff8SChuck Lever else 2908c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (autobind) via %s\n", 2909c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2910c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 2911edb267a6SChuck Lever 2912bc25571eS\"Talpey, Thomas\ if (try_module_get(THIS_MODULE)) 2913c8541ecdSChuck Lever return xprt; 29140a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL); 29150a68b0beSJ. Bruce Fields out_err: 2916315f3812SKinglong Mee xs_xprt_free(xprt); 29170a68b0beSJ. Bruce Fields return ret; 2918a246b010SChuck Lever } 2919a246b010SChuck Lever 29202881ae74STrond Myklebust static const struct rpc_timeout xs_tcp_default_timeout = { 29212881ae74STrond Myklebust .to_initval = 60 * HZ, 29222881ae74STrond Myklebust .to_maxval = 60 * HZ, 29232881ae74STrond Myklebust .to_retries = 2, 29242881ae74STrond Myklebust }; 29252881ae74STrond Myklebust 29269903cd1cSChuck Lever /** 29279903cd1cSChuck Lever * xs_setup_tcp - Set up transport to use a TCP socket 292896802a09SFrank van Maarseveen * @args: rpc transport creation arguments 29299903cd1cSChuck Lever * 29309903cd1cSChuck Lever */ 2931483066d6SAdrian Bunk static struct rpc_xprt *xs_setup_tcp(struct xprt_create *args) 2932a246b010SChuck Lever { 29338f9d5b1aSChuck Lever struct sockaddr *addr = args->dstaddr; 2934c8541ecdSChuck Lever struct rpc_xprt *xprt; 2935c8475461SChuck Lever struct sock_xprt *transport; 29360a68b0beSJ. Bruce Fields struct rpc_xprt *ret; 2937b7993cebSTrond Myklebust unsigned int max_slot_table_size = xprt_max_tcp_slot_table_entries; 2938b7993cebSTrond Myklebust 2939b7993cebSTrond Myklebust if (args->flags & XPRT_CREATE_INFINITE_SLOTS) 2940b7993cebSTrond Myklebust max_slot_table_size = RPC_MAX_SLOT_TABLE_LIMIT; 2941a246b010SChuck Lever 2942d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries, 2943b7993cebSTrond Myklebust max_slot_table_size); 2944c8541ecdSChuck Lever if (IS_ERR(xprt)) 2945c8541ecdSChuck Lever return xprt; 2946c8475461SChuck Lever transport = container_of(xprt, struct sock_xprt, xprt); 2947a246b010SChuck Lever 2948ec739ef0SChuck Lever xprt->prot = IPPROTO_TCP; 2949808012fbSChuck Lever xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32); 2950808012fbSChuck Lever xprt->max_payload = RPC_MAX_FRAGMENT_SIZE; 2951a246b010SChuck Lever 295203bf4b70SChuck Lever xprt->bind_timeout = XS_BIND_TO; 295303bf4b70SChuck Lever xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 295403bf4b70SChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO; 2955a246b010SChuck Lever 2956262965f5SChuck Lever xprt->ops = &xs_tcp_ops; 2957ba7392bbSTrond Myklebust xprt->timeout = &xs_tcp_default_timeout; 2958a246b010SChuck Lever 29593851f1cdSTrond Myklebust xprt->max_reconnect_timeout = xprt->timeout->to_maxval; 29607196dbb0STrond Myklebust xprt->connect_timeout = xprt->timeout->to_initval * 29617196dbb0STrond Myklebust (xprt->timeout->to_retries + 1); 29623851f1cdSTrond Myklebust 2963c50b8ee0STrond Myklebust INIT_WORK(&transport->recv_worker, xs_stream_data_receive_workfn); 2964edc1b01cSTrond Myklebust INIT_DELAYED_WORK(&transport->connect_worker, xs_tcp_setup_socket); 2965edc1b01cSTrond Myklebust 29668f9d5b1aSChuck Lever switch (addr->sa_family) { 29678f9d5b1aSChuck Lever case AF_INET: 29688f9d5b1aSChuck Lever if (((struct sockaddr_in *)addr)->sin_port != htons(0)) 29698f9d5b1aSChuck Lever xprt_set_bound(xprt); 29708f9d5b1aSChuck Lever 29719dc3b095SChuck Lever xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP); 29728f9d5b1aSChuck Lever break; 29738f9d5b1aSChuck Lever case AF_INET6: 29748f9d5b1aSChuck Lever if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0)) 29758f9d5b1aSChuck Lever xprt_set_bound(xprt); 29768f9d5b1aSChuck Lever 29779dc3b095SChuck Lever xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP6); 29788f9d5b1aSChuck Lever break; 29798f9d5b1aSChuck Lever default: 29800a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT); 29810a68b0beSJ. Bruce Fields goto out_err; 29828f9d5b1aSChuck Lever } 29838f9d5b1aSChuck Lever 2984c740eff8SChuck Lever if (xprt_bound(xprt)) 2985c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (port %s) via %s\n", 2986c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2987c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT], 2988c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 2989c740eff8SChuck Lever else 2990c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (autobind) via %s\n", 2991c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2992c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 2993c740eff8SChuck Lever 2994bc25571eS\"Talpey, Thomas\ if (try_module_get(THIS_MODULE)) 2995c8541ecdSChuck Lever return xprt; 29960a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL); 29970a68b0beSJ. Bruce Fields out_err: 2998315f3812SKinglong Mee xs_xprt_free(xprt); 29990a68b0beSJ. Bruce Fields return ret; 3000a246b010SChuck Lever } 3001282b32e1SChuck Lever 3002f300babaSAlexandros Batsakis /** 3003f300babaSAlexandros Batsakis * xs_setup_bc_tcp - Set up transport to use a TCP backchannel socket 3004f300babaSAlexandros Batsakis * @args: rpc transport creation arguments 3005f300babaSAlexandros Batsakis * 3006f300babaSAlexandros Batsakis */ 3007f300babaSAlexandros Batsakis static struct rpc_xprt *xs_setup_bc_tcp(struct xprt_create *args) 3008f300babaSAlexandros Batsakis { 3009f300babaSAlexandros Batsakis struct sockaddr *addr = args->dstaddr; 3010f300babaSAlexandros Batsakis struct rpc_xprt *xprt; 3011f300babaSAlexandros Batsakis struct sock_xprt *transport; 3012f300babaSAlexandros Batsakis struct svc_sock *bc_sock; 30130a68b0beSJ. Bruce Fields struct rpc_xprt *ret; 3014f300babaSAlexandros Batsakis 3015d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries, 3016d9ba131dSTrond Myklebust xprt_tcp_slot_table_entries); 3017f300babaSAlexandros Batsakis if (IS_ERR(xprt)) 3018f300babaSAlexandros Batsakis return xprt; 3019f300babaSAlexandros Batsakis transport = container_of(xprt, struct sock_xprt, xprt); 3020f300babaSAlexandros Batsakis 3021f300babaSAlexandros Batsakis xprt->prot = IPPROTO_TCP; 3022f300babaSAlexandros Batsakis xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32); 3023f300babaSAlexandros Batsakis xprt->max_payload = RPC_MAX_FRAGMENT_SIZE; 3024f300babaSAlexandros Batsakis xprt->timeout = &xs_tcp_default_timeout; 3025f300babaSAlexandros Batsakis 3026f300babaSAlexandros Batsakis /* backchannel */ 3027f300babaSAlexandros Batsakis xprt_set_bound(xprt); 3028f300babaSAlexandros Batsakis xprt->bind_timeout = 0; 3029f300babaSAlexandros Batsakis xprt->reestablish_timeout = 0; 3030f300babaSAlexandros Batsakis xprt->idle_timeout = 0; 3031f300babaSAlexandros Batsakis 3032f300babaSAlexandros Batsakis xprt->ops = &bc_tcp_ops; 3033f300babaSAlexandros Batsakis 3034f300babaSAlexandros Batsakis switch (addr->sa_family) { 3035f300babaSAlexandros Batsakis case AF_INET: 3036f300babaSAlexandros Batsakis xs_format_peer_addresses(xprt, "tcp", 3037f300babaSAlexandros Batsakis RPCBIND_NETID_TCP); 3038f300babaSAlexandros Batsakis break; 3039f300babaSAlexandros Batsakis case AF_INET6: 3040f300babaSAlexandros Batsakis xs_format_peer_addresses(xprt, "tcp", 3041f300babaSAlexandros Batsakis RPCBIND_NETID_TCP6); 3042f300babaSAlexandros Batsakis break; 3043f300babaSAlexandros Batsakis default: 30440a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT); 30450a68b0beSJ. Bruce Fields goto out_err; 3046f300babaSAlexandros Batsakis } 3047f300babaSAlexandros Batsakis 3048f300babaSAlexandros Batsakis dprintk("RPC: set up xprt to %s (port %s) via %s\n", 3049f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_ADDR], 3050f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_PORT], 3051f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_PROTO]); 3052f300babaSAlexandros Batsakis 3053f300babaSAlexandros Batsakis /* 305499de8ea9SJ. Bruce Fields * Once we've associated a backchannel xprt with a connection, 305528303ca3SWeng Meiling * we want to keep it around as long as the connection lasts, 305628303ca3SWeng Meiling * in case we need to start using it for a backchannel again; 305728303ca3SWeng Meiling * this reference won't be dropped until bc_xprt is destroyed. 305899de8ea9SJ. Bruce Fields */ 305999de8ea9SJ. Bruce Fields xprt_get(xprt); 306099de8ea9SJ. Bruce Fields args->bc_xprt->xpt_bc_xprt = xprt; 306199de8ea9SJ. Bruce Fields xprt->bc_xprt = args->bc_xprt; 306299de8ea9SJ. Bruce Fields bc_sock = container_of(args->bc_xprt, struct svc_sock, sk_xprt); 306399de8ea9SJ. Bruce Fields transport->sock = bc_sock->sk_sock; 306499de8ea9SJ. Bruce Fields transport->inet = bc_sock->sk_sk; 306599de8ea9SJ. Bruce Fields 306699de8ea9SJ. Bruce Fields /* 3067f300babaSAlexandros Batsakis * Since we don't want connections for the backchannel, we set 3068f300babaSAlexandros Batsakis * the xprt status to connected 3069f300babaSAlexandros Batsakis */ 3070f300babaSAlexandros Batsakis xprt_set_connected(xprt); 3071f300babaSAlexandros Batsakis 3072f300babaSAlexandros Batsakis if (try_module_get(THIS_MODULE)) 3073f300babaSAlexandros Batsakis return xprt; 3074642aab58SKinglong Mee 3075642aab58SKinglong Mee args->bc_xprt->xpt_bc_xprt = NULL; 307639a9beabSJ. Bruce Fields args->bc_xprt->xpt_bc_xps = NULL; 307799de8ea9SJ. Bruce Fields xprt_put(xprt); 30780a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL); 30790a68b0beSJ. Bruce Fields out_err: 3080315f3812SKinglong Mee xs_xprt_free(xprt); 30810a68b0beSJ. Bruce Fields return ret; 3082f300babaSAlexandros Batsakis } 3083f300babaSAlexandros Batsakis 3084176e21eeSChuck Lever static struct xprt_class xs_local_transport = { 3085176e21eeSChuck Lever .list = LIST_HEAD_INIT(xs_local_transport.list), 3086176e21eeSChuck Lever .name = "named UNIX socket", 3087176e21eeSChuck Lever .owner = THIS_MODULE, 3088176e21eeSChuck Lever .ident = XPRT_TRANSPORT_LOCAL, 3089176e21eeSChuck Lever .setup = xs_setup_local, 3090176e21eeSChuck Lever }; 3091176e21eeSChuck Lever 3092bc25571eS\"Talpey, Thomas\ static struct xprt_class xs_udp_transport = { 3093bc25571eS\"Talpey, Thomas\ .list = LIST_HEAD_INIT(xs_udp_transport.list), 3094bc25571eS\"Talpey, Thomas\ .name = "udp", 3095bc25571eS\"Talpey, Thomas\ .owner = THIS_MODULE, 3096f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_UDP, 3097bc25571eS\"Talpey, Thomas\ .setup = xs_setup_udp, 3098bc25571eS\"Talpey, Thomas\ }; 3099bc25571eS\"Talpey, Thomas\ 3100bc25571eS\"Talpey, Thomas\ static struct xprt_class xs_tcp_transport = { 3101bc25571eS\"Talpey, Thomas\ .list = LIST_HEAD_INIT(xs_tcp_transport.list), 3102bc25571eS\"Talpey, Thomas\ .name = "tcp", 3103bc25571eS\"Talpey, Thomas\ .owner = THIS_MODULE, 3104f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_TCP, 3105bc25571eS\"Talpey, Thomas\ .setup = xs_setup_tcp, 3106bc25571eS\"Talpey, Thomas\ }; 3107bc25571eS\"Talpey, Thomas\ 3108f300babaSAlexandros Batsakis static struct xprt_class xs_bc_tcp_transport = { 3109f300babaSAlexandros Batsakis .list = LIST_HEAD_INIT(xs_bc_tcp_transport.list), 3110f300babaSAlexandros Batsakis .name = "tcp NFSv4.1 backchannel", 3111f300babaSAlexandros Batsakis .owner = THIS_MODULE, 3112f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_BC_TCP, 3113f300babaSAlexandros Batsakis .setup = xs_setup_bc_tcp, 3114f300babaSAlexandros Batsakis }; 3115f300babaSAlexandros Batsakis 3116282b32e1SChuck Lever /** 3117bc25571eS\"Talpey, Thomas\ * init_socket_xprt - set up xprtsock's sysctls, register with RPC client 3118282b32e1SChuck Lever * 3119282b32e1SChuck Lever */ 3120282b32e1SChuck Lever int init_socket_xprt(void) 3121282b32e1SChuck Lever { 31222b1bec5fSEric W. Biederman if (!sunrpc_table_header) 31230b4d4147SEric W. Biederman sunrpc_table_header = register_sysctl_table(sunrpc_table); 3124fbf76683SChuck Lever 3125176e21eeSChuck Lever xprt_register_transport(&xs_local_transport); 3126bc25571eS\"Talpey, Thomas\ xprt_register_transport(&xs_udp_transport); 3127bc25571eS\"Talpey, Thomas\ xprt_register_transport(&xs_tcp_transport); 3128f300babaSAlexandros Batsakis xprt_register_transport(&xs_bc_tcp_transport); 3129bc25571eS\"Talpey, Thomas\ 3130282b32e1SChuck Lever return 0; 3131282b32e1SChuck Lever } 3132282b32e1SChuck Lever 3133282b32e1SChuck Lever /** 3134bc25571eS\"Talpey, Thomas\ * cleanup_socket_xprt - remove xprtsock's sysctls, unregister 3135282b32e1SChuck Lever * 3136282b32e1SChuck Lever */ 3137282b32e1SChuck Lever void cleanup_socket_xprt(void) 3138282b32e1SChuck Lever { 3139fbf76683SChuck Lever if (sunrpc_table_header) { 3140fbf76683SChuck Lever unregister_sysctl_table(sunrpc_table_header); 3141fbf76683SChuck Lever sunrpc_table_header = NULL; 3142fbf76683SChuck Lever } 3143bc25571eS\"Talpey, Thomas\ 3144176e21eeSChuck Lever xprt_unregister_transport(&xs_local_transport); 3145bc25571eS\"Talpey, Thomas\ xprt_unregister_transport(&xs_udp_transport); 3146bc25571eS\"Talpey, Thomas\ xprt_unregister_transport(&xs_tcp_transport); 3147f300babaSAlexandros Batsakis xprt_unregister_transport(&xs_bc_tcp_transport); 3148282b32e1SChuck Lever } 3149cbf11071STrond Myklebust 31509bbb9e5aSRusty Russell static int param_set_uint_minmax(const char *val, 31519bbb9e5aSRusty Russell const struct kernel_param *kp, 3152cbf11071STrond Myklebust unsigned int min, unsigned int max) 3153cbf11071STrond Myklebust { 315400cfaa94SDaniel Walter unsigned int num; 3155cbf11071STrond Myklebust int ret; 3156cbf11071STrond Myklebust 3157cbf11071STrond Myklebust if (!val) 3158cbf11071STrond Myklebust return -EINVAL; 315900cfaa94SDaniel Walter ret = kstrtouint(val, 0, &num); 31609761a246SDan Carpenter if (ret) 31619761a246SDan Carpenter return ret; 31629761a246SDan Carpenter if (num < min || num > max) 3163cbf11071STrond Myklebust return -EINVAL; 3164cbf11071STrond Myklebust *((unsigned int *)kp->arg) = num; 3165cbf11071STrond Myklebust return 0; 3166cbf11071STrond Myklebust } 3167cbf11071STrond Myklebust 31689bbb9e5aSRusty Russell static int param_set_portnr(const char *val, const struct kernel_param *kp) 3169cbf11071STrond Myklebust { 3170cbf11071STrond Myklebust return param_set_uint_minmax(val, kp, 3171cbf11071STrond Myklebust RPC_MIN_RESVPORT, 3172cbf11071STrond Myklebust RPC_MAX_RESVPORT); 3173cbf11071STrond Myklebust } 3174cbf11071STrond Myklebust 31759c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_portnr = { 31769bbb9e5aSRusty Russell .set = param_set_portnr, 31779bbb9e5aSRusty Russell .get = param_get_uint, 31789bbb9e5aSRusty Russell }; 31799bbb9e5aSRusty Russell 3180cbf11071STrond Myklebust #define param_check_portnr(name, p) \ 3181cbf11071STrond Myklebust __param_check(name, p, unsigned int); 3182cbf11071STrond Myklebust 3183cbf11071STrond Myklebust module_param_named(min_resvport, xprt_min_resvport, portnr, 0644); 3184cbf11071STrond Myklebust module_param_named(max_resvport, xprt_max_resvport, portnr, 0644); 3185cbf11071STrond Myklebust 31869bbb9e5aSRusty Russell static int param_set_slot_table_size(const char *val, 31879bbb9e5aSRusty Russell const struct kernel_param *kp) 3188cbf11071STrond Myklebust { 3189cbf11071STrond Myklebust return param_set_uint_minmax(val, kp, 3190cbf11071STrond Myklebust RPC_MIN_SLOT_TABLE, 3191cbf11071STrond Myklebust RPC_MAX_SLOT_TABLE); 3192cbf11071STrond Myklebust } 3193cbf11071STrond Myklebust 31949c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_slot_table_size = { 31959bbb9e5aSRusty Russell .set = param_set_slot_table_size, 31969bbb9e5aSRusty Russell .get = param_get_uint, 31979bbb9e5aSRusty Russell }; 31989bbb9e5aSRusty Russell 3199cbf11071STrond Myklebust #define param_check_slot_table_size(name, p) \ 3200cbf11071STrond Myklebust __param_check(name, p, unsigned int); 3201cbf11071STrond Myklebust 3202d9ba131dSTrond Myklebust static int param_set_max_slot_table_size(const char *val, 3203d9ba131dSTrond Myklebust const struct kernel_param *kp) 3204d9ba131dSTrond Myklebust { 3205d9ba131dSTrond Myklebust return param_set_uint_minmax(val, kp, 3206d9ba131dSTrond Myklebust RPC_MIN_SLOT_TABLE, 3207d9ba131dSTrond Myklebust RPC_MAX_SLOT_TABLE_LIMIT); 3208d9ba131dSTrond Myklebust } 3209d9ba131dSTrond Myklebust 32109c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_max_slot_table_size = { 3211d9ba131dSTrond Myklebust .set = param_set_max_slot_table_size, 3212d9ba131dSTrond Myklebust .get = param_get_uint, 3213d9ba131dSTrond Myklebust }; 3214d9ba131dSTrond Myklebust 3215d9ba131dSTrond Myklebust #define param_check_max_slot_table_size(name, p) \ 3216d9ba131dSTrond Myklebust __param_check(name, p, unsigned int); 3217d9ba131dSTrond Myklebust 3218cbf11071STrond Myklebust module_param_named(tcp_slot_table_entries, xprt_tcp_slot_table_entries, 3219cbf11071STrond Myklebust slot_table_size, 0644); 3220d9ba131dSTrond Myklebust module_param_named(tcp_max_slot_table_entries, xprt_max_tcp_slot_table_entries, 3221d9ba131dSTrond Myklebust max_slot_table_size, 0644); 3222cbf11071STrond Myklebust module_param_named(udp_slot_table_entries, xprt_udp_slot_table_entries, 3223cbf11071STrond Myklebust slot_table_size, 0644); 3224