156 static void svc_set_cmsg_data(struct svc_rqst *rqstp, struct cmsghdr *cmh)  in svc_set_cmsg_data()  argument
159 		container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt);  in svc_set_cmsg_data()
168 				 svc_daddr_in(rqstp)->sin_addr.s_addr;  in svc_set_cmsg_data()
175 			struct sockaddr_in6 *daddr = svc_daddr_in6(rqstp);  in svc_set_cmsg_data()
187 static int svc_sock_result_payload(struct svc_rqst *rqstp, unsigned int offset,  in svc_sock_result_payload()  argument
301 static ssize_t svc_tcp_read_msg(struct svc_rqst *rqstp, size_t buflen,  in svc_tcp_read_msg()  argument
305 		container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt);  in svc_tcp_read_msg()
306 	struct bio_vec *bvec = rqstp->rq_bvec;  in svc_tcp_read_msg()
315 		bvec_set_page(&bvec[i], rqstp->rq_pages[i], PAGE_SIZE, 0);  in svc_tcp_read_msg()
316 	rqstp->rq_respages = &rqstp->rq_pages[i];  in svc_tcp_read_msg()
317 	rqstp->rq_next_page = rqstp->rq_respages + 1;  in svc_tcp_read_msg()
354 static void svc_sock_secure_port(struct svc_rqst *rqstp)  in svc_sock_secure_port()  argument
356 	if (svc_port_is_privileged(svc_addr(rqstp)))  in svc_sock_secure_port()
357 		set_bit(RQ_SECURE, &rqstp->rq_flags);  in svc_sock_secure_port()
359 		clear_bit(RQ_SECURE, &rqstp->rq_flags);  in svc_sock_secure_port()
500 static int svc_udp_get_dest_address4(struct svc_rqst *rqstp,  in svc_udp_get_dest_address4()  argument
504 	struct sockaddr_in *daddr = svc_daddr_in(rqstp);  in svc_udp_get_dest_address4()
517 static int svc_udp_get_dest_address6(struct svc_rqst *rqstp,  in svc_udp_get_dest_address6()  argument
521 	struct sockaddr_in6 *daddr = svc_daddr_in6(rqstp);  in svc_udp_get_dest_address6()
539 static int svc_udp_get_dest_address(struct svc_rqst *rqstp,  in svc_udp_get_dest_address()  argument
544 		return svc_udp_get_dest_address4(rqstp, cmh);  in svc_udp_get_dest_address()
546 		return svc_udp_get_dest_address6(rqstp, cmh);  in svc_udp_get_dest_address()
562 static int svc_udp_recvfrom(struct svc_rqst *rqstp)  in svc_udp_recvfrom()  argument
565 		container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt);  in svc_udp_recvfrom()
574 		.msg_name = svc_addr(rqstp),  in svc_udp_recvfrom()
602 	len = svc_addr_len(svc_addr(rqstp));  in svc_udp_recvfrom()
603 	rqstp->rq_addrlen = len;  in svc_udp_recvfrom()
613 	rqstp->rq_arg.len = len;  in svc_udp_recvfrom()
616 	rqstp->rq_prot = IPPROTO_UDP;  in svc_udp_recvfrom()
618 	if (!svc_udp_get_dest_address(rqstp, cmh))  in svc_udp_recvfrom()
620 	rqstp->rq_daddrlen = svc_addr_len(svc_daddr(rqstp));  in svc_udp_recvfrom()
625 		if (csum_partial_copy_to_xdr(&rqstp->rq_arg, skb))  in svc_udp_recvfrom()
631 		rqstp->rq_arg.head[0].iov_base = skb->data;  in svc_udp_recvfrom()
632 		rqstp->rq_arg.head[0].iov_len = len;  in svc_udp_recvfrom()
635 		rqstp->rq_xprt_ctxt = skb;  in svc_udp_recvfrom()
638 	rqstp->rq_arg.page_base = 0;  in svc_udp_recvfrom()
639 	if (len <= rqstp->rq_arg.head[0].iov_len) {  in svc_udp_recvfrom()
640 		rqstp->rq_arg.head[0].iov_len = len;  in svc_udp_recvfrom()
641 		rqstp->rq_arg.page_len = 0;  in svc_udp_recvfrom()
642 		rqstp->rq_respages = rqstp->rq_pages+1;  in svc_udp_recvfrom()
644 		rqstp->rq_arg.page_len = len - rqstp->rq_arg.head[0].iov_len;  in svc_udp_recvfrom()
645 		rqstp->rq_respages = rqstp->rq_pages + 1 +  in svc_udp_recvfrom()
646 			DIV_ROUND_UP(rqstp->rq_arg.page_len, PAGE_SIZE);  in svc_udp_recvfrom()
648 	rqstp->rq_next_page = rqstp->rq_respages+1;  in svc_udp_recvfrom()
653 	svc_sock_secure_port(rqstp);  in svc_udp_recvfrom()
654 	svc_xprt_received(rqstp->rq_xprt);  in svc_udp_recvfrom()
673 	svc_xprt_received(rqstp->rq_xprt);  in svc_udp_recvfrom()
686 static int svc_udp_sendto(struct svc_rqst *rqstp)  in svc_udp_sendto()  argument
688 	struct svc_xprt *xprt = rqstp->rq_xprt;  in svc_udp_sendto()
690 	struct xdr_buf *xdr = &rqstp->rq_res;  in svc_udp_sendto()
697 		.msg_name	= &rqstp->rq_addr,  in svc_udp_sendto()
698 		.msg_namelen	= rqstp->rq_addrlen,  in svc_udp_sendto()
706 	svc_udp_release_ctxt(xprt, rqstp->rq_xprt_ctxt);  in svc_udp_sendto()
707 	rqstp->rq_xprt_ctxt = NULL;  in svc_udp_sendto()
709 	svc_set_cmsg_data(rqstp, cmh);  in svc_udp_sendto()
716 	count = xdr_buf_to_bvec(rqstp->rq_bvec,  in svc_udp_sendto()
717 				ARRAY_SIZE(rqstp->rq_bvec), xdr);  in svc_udp_sendto()
719 	iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, rqstp->rq_bvec,  in svc_udp_sendto()
720 		      count, rqstp->rq_res.len);  in svc_udp_sendto()
724 		iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, rqstp->rq_bvec,  in svc_udp_sendto()
725 			      count, rqstp->rq_res.len);  in svc_udp_sendto()
949 				    struct svc_rqst *rqstp)  in svc_tcp_restore_pages()  argument
958 		if (rqstp->rq_pages[i] != NULL)  in svc_tcp_restore_pages()
959 			put_page(rqstp->rq_pages[i]);  in svc_tcp_restore_pages()
961 		rqstp->rq_pages[i] = svsk->sk_pages[i];  in svc_tcp_restore_pages()
964 	rqstp->rq_arg.head[0].iov_base = page_address(rqstp->rq_pages[0]);  in svc_tcp_restore_pages()
968 static void svc_tcp_save_pages(struct svc_sock *svsk, struct svc_rqst *rqstp)  in svc_tcp_save_pages()  argument
977 		svsk->sk_pages[i] = rqstp->rq_pages[i];  in svc_tcp_save_pages()
978 		rqstp->rq_pages[i] = NULL;  in svc_tcp_save_pages()
1007 				   struct svc_rqst *rqstp)  in svc_tcp_read_marker()  argument
1046 static int receive_cb_reply(struct svc_sock *svsk, struct svc_rqst *rqstp)  in receive_cb_reply()  argument
1051 	__be32 *p = (__be32 *)rqstp->rq_arg.head[0].iov_base;  in receive_cb_reply()
1072 	src = &rqstp->rq_arg.head[0];  in receive_cb_reply()
1076 	xprt_complete_rqst(req->rq_task, rqstp->rq_arg.len);  in receive_cb_reply()
1077 	rqstp->rq_arg.len = 0;  in receive_cb_reply()
1119 static int svc_tcp_recvfrom(struct svc_rqst *rqstp)  in svc_tcp_recvfrom()  argument
1122 		container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt);  in svc_tcp_recvfrom()
1130 	len = svc_tcp_read_marker(svsk, rqstp);  in svc_tcp_recvfrom()
1134 	base = svc_tcp_restore_pages(svsk, rqstp);  in svc_tcp_recvfrom()
1136 	len = svc_tcp_read_msg(rqstp, base + want, base);  in svc_tcp_recvfrom()
1147 	rqstp->rq_arg.len = svsk->sk_datalen;  in svc_tcp_recvfrom()
1148 	rqstp->rq_arg.page_base = 0;  in svc_tcp_recvfrom()
1149 	if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len) {  in svc_tcp_recvfrom()
1150 		rqstp->rq_arg.head[0].iov_len = rqstp->rq_arg.len;  in svc_tcp_recvfrom()
1151 		rqstp->rq_arg.page_len = 0;  in svc_tcp_recvfrom()
1153 		rqstp->rq_arg.page_len = rqstp->rq_arg.len - rqstp->rq_arg.head[0].iov_len;  in svc_tcp_recvfrom()
1155 	rqstp->rq_xprt_ctxt   = NULL;  in svc_tcp_recvfrom()
1156 	rqstp->rq_prot	      = IPPROTO_TCP;  in svc_tcp_recvfrom()
1158 		set_bit(RQ_LOCAL, &rqstp->rq_flags);  in svc_tcp_recvfrom()
1160 		clear_bit(RQ_LOCAL, &rqstp->rq_flags);  in svc_tcp_recvfrom()
1162 	p = (__be32 *)rqstp->rq_arg.head[0].iov_base;  in svc_tcp_recvfrom()
1165 		len = receive_cb_reply(svsk, rqstp);  in svc_tcp_recvfrom()
1174 	svc_xprt_copy_addrs(rqstp, &svsk->sk_xprt);  in svc_tcp_recvfrom()
1178 	svc_sock_secure_port(rqstp);  in svc_tcp_recvfrom()
1179 	svc_xprt_received(rqstp->rq_xprt);  in svc_tcp_recvfrom()
1180 	return rqstp->rq_arg.len;  in svc_tcp_recvfrom()
1183 	svc_tcp_save_pages(svsk, rqstp);  in svc_tcp_recvfrom()
1211 	svc_xprt_received(rqstp->rq_xprt);  in svc_tcp_recvfrom()
1220 static int svc_tcp_sendmsg(struct svc_sock *svsk, struct svc_rqst *rqstp,  in svc_tcp_sendmsg()  argument
1240 	bvec_set_virt(rqstp->rq_bvec, buf, sizeof(marker));  in svc_tcp_sendmsg()
1242 	count = xdr_buf_to_bvec(rqstp->rq_bvec + 1,  in svc_tcp_sendmsg()
1243 				ARRAY_SIZE(rqstp->rq_bvec) - 1, &rqstp->rq_res);  in svc_tcp_sendmsg()
1245 	iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, rqstp->rq_bvec,  in svc_tcp_sendmsg()
1246 		      1 + count, sizeof(marker) + rqstp->rq_res.len);  in svc_tcp_sendmsg()
1264 static int svc_tcp_sendto(struct svc_rqst *rqstp)  in svc_tcp_sendto()  argument
1266 	struct svc_xprt *xprt = rqstp->rq_xprt;  in svc_tcp_sendto()
1268 	struct xdr_buf *xdr = &rqstp->rq_res;  in svc_tcp_sendto()
1274 	svc_tcp_release_ctxt(xprt, rqstp->rq_xprt_ctxt);  in svc_tcp_sendto()
1275 	rqstp->rq_xprt_ctxt = NULL;  in svc_tcp_sendto()
1280 	err = svc_tcp_sendmsg(svsk, rqstp, marker, &sent);  in svc_tcp_sendto()