33 			   struct sk_msg *msg, u32 apply_bytes, int flags)  in bpf_tcp_ingress()  argument
46 	tmp->sg.start = msg->sg.start;  in bpf_tcp_ingress()
47 	i = msg->sg.start;  in bpf_tcp_ingress()
49 		sge = sk_msg_elem(msg, i);  in bpf_tcp_ingress()
59 		sk_msg_xfer(tmp, msg, i, size);  in bpf_tcp_ingress()
73 	} while (i != msg->sg.end);  in bpf_tcp_ingress()
76 		msg->sg.start = i;  in bpf_tcp_ingress()
88 static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes,  in tcp_bpf_push()  argument
102 		sge = sk_msg_elem(msg, msg->sg.start);  in tcp_bpf_push()
115 		if (size < sge->length && msg->sg.start != msg->sg.end)  in tcp_bpf_push()
126 		msg->sg.size -= ret;  in tcp_bpf_push()
138 			sk_msg_iter_next(msg, start);  in tcp_bpf_push()
140 			if (msg->sg.start == msg->sg.end)  in tcp_bpf_push()
150 static int tcp_bpf_push_locked(struct sock *sk, struct sk_msg *msg,  in tcp_bpf_push_locked()  argument
156 	ret = tcp_bpf_push(sk, msg, apply_bytes, flags, uncharge);  in tcp_bpf_push_locked()
162 			  struct sk_msg *msg, u32 bytes, int flags)  in tcp_bpf_sendmsg_redir()  argument
170 	ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes, flags) :  in tcp_bpf_sendmsg_redir()
171 			tcp_bpf_push_locked(sk, msg, bytes, flags, false);  in tcp_bpf_sendmsg_redir()
219 				  struct msghdr *msg,  in tcp_bpf_recvmsg_parser()  argument
231 		return inet_recv_error(sk, msg, len, addr_len);  in tcp_bpf_recvmsg_parser()
238 		return tcp_recvmsg(sk, msg, len, flags, addr_len);  in tcp_bpf_recvmsg_parser()
262 	copied = sk_msg_recvmsg(sk, psock, msg, len, flags);  in tcp_bpf_recvmsg_parser()
331 static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,  in tcp_bpf_recvmsg()  argument
338 		return inet_recv_error(sk, msg, len, addr_len);  in tcp_bpf_recvmsg()
345 		return tcp_recvmsg(sk, msg, len, flags, addr_len);  in tcp_bpf_recvmsg()
349 		return tcp_recvmsg(sk, msg, len, flags, addr_len);  in tcp_bpf_recvmsg()
353 	copied = sk_msg_recvmsg(sk, psock, msg, len, flags);  in tcp_bpf_recvmsg()
369 			return tcp_recvmsg(sk, msg, len, flags, addr_len);  in tcp_bpf_recvmsg()
382 				struct sk_msg *msg, int *copied, int flags)  in tcp_bpf_send_verdict()  argument
384 	bool cork = false, enospc = sk_msg_full(msg), redir_ingress;  in tcp_bpf_send_verdict()
397 		delta = msg->sg.size;  in tcp_bpf_send_verdict()
398 		psock->eval = sk_psock_msg_verdict(sk, psock, msg);  in tcp_bpf_send_verdict()
399 		delta -= msg->sg.size;  in tcp_bpf_send_verdict()
402 	if (msg->cork_bytes &&  in tcp_bpf_send_verdict()
403 	    msg->cork_bytes > msg->sg.size && !enospc) {  in tcp_bpf_send_verdict()
404 		psock->cork_bytes = msg->cork_bytes - msg->sg.size;  in tcp_bpf_send_verdict()
411 		memcpy(psock->cork, msg, sizeof(*msg));  in tcp_bpf_send_verdict()
415 	tosend = msg->sg.size;  in tcp_bpf_send_verdict()
422 		ret = tcp_bpf_push(sk, msg, tosend, flags, true);  in tcp_bpf_send_verdict()
424 			*copied -= sk_msg_free(sk, msg);  in tcp_bpf_send_verdict()
443 		sk_msg_return(sk, msg, tosend);  in tcp_bpf_send_verdict()
446 		origsize = msg->sg.size;  in tcp_bpf_send_verdict()
448 					    msg, tosend, flags);  in tcp_bpf_send_verdict()
449 		sent = origsize - msg->sg.size;  in tcp_bpf_send_verdict()
456 			int free = sk_msg_free_nocharge(sk, msg);  in tcp_bpf_send_verdict()
462 			sk_msg_free(sk, msg);  in tcp_bpf_send_verdict()
463 			kfree(msg);  in tcp_bpf_send_verdict()
464 			msg = NULL;  in tcp_bpf_send_verdict()
470 		sk_msg_free_partial(sk, msg, tosend);  in tcp_bpf_send_verdict()
484 		if (msg &&  in tcp_bpf_send_verdict()
485 		    msg->sg.data[msg->sg.start].page_link &&  in tcp_bpf_send_verdict()
486 		    msg->sg.data[msg->sg.start].length) {  in tcp_bpf_send_verdict()
495 static int tcp_bpf_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)  in tcp_bpf_sendmsg()  argument
504 	flags = (msg->msg_flags & ~MSG_SENDPAGE_DECRYPTED);  in tcp_bpf_sendmsg()
509 		return tcp_sendmsg(sk, msg, size);  in tcp_bpf_sendmsg()
512 	timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);  in tcp_bpf_sendmsg()
513 	while (msg_data_left(msg)) {  in tcp_bpf_sendmsg()
522 		copy = msg_data_left(msg);  in tcp_bpf_sendmsg()
541 		err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, msg_tx,  in tcp_bpf_sendmsg()
577 		err = sk_stream_error(sk, msg->msg_flags, err);  in tcp_bpf_sendmsg()