842 static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk,  in bpf_exec_tx_verdict()  argument
861 			*copied -= sk_msg_free(sk, msg);  in bpf_exec_tx_verdict()
870 	enospc = sk_msg_full(msg);  in bpf_exec_tx_verdict()
872 		delta = msg->sg.size;  in bpf_exec_tx_verdict()
873 		psock->eval = sk_psock_msg_verdict(sk, psock, msg);  in bpf_exec_tx_verdict()
874 		delta -= msg->sg.size;  in bpf_exec_tx_verdict()
876 	if (msg->cork_bytes && msg->cork_bytes > msg->sg.size &&  in bpf_exec_tx_verdict()
881 	msg->cork_bytes = 0;  in bpf_exec_tx_verdict()
882 	send = msg->sg.size;  in bpf_exec_tx_verdict()
883 	if (msg->apply_bytes && msg->apply_bytes < send)  in bpf_exec_tx_verdict()
884 		send = msg->apply_bytes;  in bpf_exec_tx_verdict()
890 			*copied -= sk_msg_free(sk, msg);  in bpf_exec_tx_verdict()
899 		memcpy(&msg_redir, msg, sizeof(*msg));  in bpf_exec_tx_verdict()
900 		if (msg->apply_bytes < send)  in bpf_exec_tx_verdict()
901 			msg->apply_bytes = 0;  in bpf_exec_tx_verdict()
903 			msg->apply_bytes -= send;  in bpf_exec_tx_verdict()
904 		sk_msg_return_zero(sk, msg, send);  in bpf_exec_tx_verdict()
905 		msg->sg.size -= send;  in bpf_exec_tx_verdict()
912 			msg->sg.size = 0;  in bpf_exec_tx_verdict()
914 		if (msg->sg.size == 0)  in bpf_exec_tx_verdict()
919 		sk_msg_free_partial(sk, msg, send);  in bpf_exec_tx_verdict()
920 		if (msg->apply_bytes < send)  in bpf_exec_tx_verdict()
921 			msg->apply_bytes = 0;  in bpf_exec_tx_verdict()
923 			msg->apply_bytes -= send;  in bpf_exec_tx_verdict()
924 		if (msg->sg.size == 0)  in bpf_exec_tx_verdict()
935 			msg = &rec->msg_plaintext;  in bpf_exec_tx_verdict()
936 			if (!msg->apply_bytes)  in bpf_exec_tx_verdict()
974 static int tls_sw_sendmsg_splice(struct sock *sk, struct msghdr *msg,  in tls_sw_sendmsg_splice()  argument
984 		part = iov_iter_extract_pages(&msg->msg_iter, &pages,  in tls_sw_sendmsg_splice()
990 			iov_iter_revert(&msg->msg_iter, part);  in tls_sw_sendmsg_splice()
1005 static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg,  in tls_sw_sendmsg_locked()  argument
1008 	long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);  in tls_sw_sendmsg_locked()
1014 	bool is_kvec = iov_iter_is_kvec(&msg->msg_iter);  in tls_sw_sendmsg_locked()
1015 	bool eor = !(msg->msg_flags & MSG_MORE);  in tls_sw_sendmsg_locked()
1028 	if (!eor && (msg->msg_flags & MSG_EOR))  in tls_sw_sendmsg_locked()
1031 	if (unlikely(msg->msg_controllen)) {  in tls_sw_sendmsg_locked()
1032 		ret = tls_process_cmsg(sk, msg, &record_type);  in tls_sw_sendmsg_locked()
1041 	while (msg_data_left(msg)) {  in tls_sw_sendmsg_locked()
1061 		try_to_copy = msg_data_left(msg);  in tls_sw_sendmsg_locked()
1088 		if (try_to_copy && (msg->msg_flags & MSG_SPLICE_PAGES)) {  in tls_sw_sendmsg_locked()
1089 			ret = tls_sw_sendmsg_splice(sk, msg, msg_pl,  in tls_sw_sendmsg_locked()
1106 			ret = sk_msg_zerocopy_from_iter(sk, &msg->msg_iter,  in tls_sw_sendmsg_locked()
1117 						  msg->msg_flags);  in tls_sw_sendmsg_locked()
1132 			iov_iter_revert(&msg->msg_iter,  in tls_sw_sendmsg_locked()
1156 			ret = sk_msg_memcopy_from_iter(sk, &msg->msg_iter,  in tls_sw_sendmsg_locked()
1171 						  msg->msg_flags);  in tls_sw_sendmsg_locked()
1218 		tls_tx_records(sk, msg->msg_flags);  in tls_sw_sendmsg_locked()
1222 	ret = sk_stream_error(sk, msg->msg_flags, ret);  in tls_sw_sendmsg_locked()
1226 int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)  in tls_sw_sendmsg()  argument
1231 	if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |  in tls_sw_sendmsg()
1240 	ret = tls_sw_sendmsg_locked(sk, msg, size);  in tls_sw_sendmsg()
1635 	       struct msghdr *msg, struct tls_decrypt_arg *darg)  in tls_decrypt_sw()  argument
1642 	err = tls_decrypt_sg(sk, &msg->msg_iter, NULL, darg);  in tls_decrypt_sw()
1657 		return tls_decrypt_sw(sk, tls_ctx, msg, darg);  in tls_decrypt_sw()
1674 tls_decrypt_device(struct sock *sk, struct msghdr *msg,  in tls_decrypt_device()  argument
1716 		err = skb_copy_datagram_msg(darg->skb, off, msg, len);  in tls_decrypt_device()
1723 static int tls_rx_one_record(struct sock *sk, struct msghdr *msg,  in tls_rx_one_record()  argument
1731 	err = tls_decrypt_device(sk, msg, tls_ctx, darg);  in tls_rx_one_record()
1733 		err = tls_decrypt_sw(sk, tls_ctx, msg, darg);  in tls_rx_one_record()
1752 static int tls_record_content_type(struct msghdr *msg, struct tls_msg *tlm,  in tls_record_content_type()  argument
1762 		err = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE,  in tls_record_content_type()
1765 			if (err || msg->msg_flags & MSG_CTRUNC)  in tls_record_content_type()
1786 			   struct msghdr *msg,  in process_rx_list()  argument
1802 		err = tls_record_content_type(msg, tlm, control);  in process_rx_list()
1820 		err = tls_record_content_type(msg, tlm, control);  in process_rx_list()
1825 					    msg, chunk);  in process_rx_list()
1847 		if (msg)  in process_rx_list()
1848 			msg->msg_flags |= MSG_EOR;  in process_rx_list()
1951 		   struct msghdr *msg,  in tls_sw_recvmsg()  argument
1969 	bool is_kvec = iov_iter_is_kvec(&msg->msg_iter);  in tls_sw_recvmsg()
1977 		return sock_recv_errqueue(sk, msg, len, SOL_IP, IP_RECVERR);  in tls_sw_recvmsg()
1991 	err = process_rx_list(ctx, msg, &control, 0, len, is_peek, &rx_more);  in tls_sw_recvmsg()
2013 				chunk = sk_msg_recvmsg(sk, psock, msg, len,  in tls_sw_recvmsg()
2041 		err = tls_rx_one_record(sk, msg, &darg);  in tls_sw_recvmsg()
2056 		err = tls_record_content_type(msg, tls_msg(darg.skb), &control);  in tls_sw_recvmsg()
2110 						    msg, chunk);  in tls_sw_recvmsg()
2134 		msg->msg_flags |= MSG_EOR;  in tls_sw_recvmsg()
2156 			err = process_rx_list(ctx, msg, &control, copied + peeked,  in tls_sw_recvmsg()
2159 			err = process_rx_list(ctx, msg, &control, 0,  in tls_sw_recvmsg()