13 static bool sk_msg_try_coalesce_ok(struct sk_msg *msg, int elem_first_coalesce)  in sk_msg_try_coalesce_ok()  argument
15 	if (msg->sg.end > msg->sg.start &&  in sk_msg_try_coalesce_ok()
16 	    elem_first_coalesce < msg->sg.end)  in sk_msg_try_coalesce_ok()
19 	if (msg->sg.end < msg->sg.start &&  in sk_msg_try_coalesce_ok()
20 	    (elem_first_coalesce > msg->sg.start ||  in sk_msg_try_coalesce_ok()
21 	     elem_first_coalesce < msg->sg.end))  in sk_msg_try_coalesce_ok()
27 int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,  in sk_msg_alloc()  argument
31 	u32 osize = msg->sg.size;  in sk_msg_alloc()
34 	len -= msg->sg.size;  in sk_msg_alloc()
52 		i = msg->sg.end;  in sk_msg_alloc()
54 		sge = &msg->sg.data[i];  in sk_msg_alloc()
56 		if (sk_msg_try_coalesce_ok(msg, elem_first_coalesce) &&  in sk_msg_alloc()
61 			if (sk_msg_full(msg)) {  in sk_msg_alloc()
66 			sge = &msg->sg.data[msg->sg.end];  in sk_msg_alloc()
70 			sk_msg_iter_next(msg, end);  in sk_msg_alloc()
74 		msg->sg.size += use;  in sk_msg_alloc()
82 	sk_msg_trim(sk, msg, osize);  in sk_msg_alloc()
138 void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes)  in sk_msg_return_zero()  argument
140 	int i = msg->sg.start;  in sk_msg_return_zero()
143 		struct scatterlist *sge = sk_msg_elem(msg, i);  in sk_msg_return_zero()
157 	} while (bytes && i != msg->sg.end);  in sk_msg_return_zero()
158 	msg->sg.start = i;  in sk_msg_return_zero()
162 void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes)  in sk_msg_return()  argument
164 	int i = msg->sg.start;  in sk_msg_return()
167 		struct scatterlist *sge = &msg->sg.data[i];  in sk_msg_return()
173 	} while (i != msg->sg.end);  in sk_msg_return()
177 static int sk_msg_free_elem(struct sock *sk, struct sk_msg *msg, u32 i,  in sk_msg_free_elem()  argument
180 	struct scatterlist *sge = sk_msg_elem(msg, i);  in sk_msg_free_elem()
184 	if (!msg->skb) {  in sk_msg_free_elem()
193 static int __sk_msg_free(struct sock *sk, struct sk_msg *msg, u32 i,  in __sk_msg_free()  argument
196 	struct scatterlist *sge = sk_msg_elem(msg, i);  in __sk_msg_free()
199 	while (msg->sg.size) {  in __sk_msg_free()
200 		msg->sg.size -= sge->length;  in __sk_msg_free()
201 		freed += sk_msg_free_elem(sk, msg, i, charge);  in __sk_msg_free()
203 		sk_msg_check_to_free(msg, i, msg->sg.size);  in __sk_msg_free()
204 		sge = sk_msg_elem(msg, i);  in __sk_msg_free()
206 	consume_skb(msg->skb);  in __sk_msg_free()
207 	sk_msg_init(msg);  in __sk_msg_free()
211 int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg)  in sk_msg_free_nocharge()  argument
213 	return __sk_msg_free(sk, msg, msg->sg.start, false);  in sk_msg_free_nocharge()
217 int sk_msg_free(struct sock *sk, struct sk_msg *msg)  in sk_msg_free()  argument
219 	return __sk_msg_free(sk, msg, msg->sg.start, true);  in sk_msg_free()
223 static void __sk_msg_free_partial(struct sock *sk, struct sk_msg *msg,  in __sk_msg_free_partial()  argument
227 	u32 i = msg->sg.start;  in __sk_msg_free_partial()
230 		sge = sk_msg_elem(msg, i);  in __sk_msg_free_partial()
238 			msg->sg.size -= bytes;  in __sk_msg_free_partial()
242 		msg->sg.size -= sge->length;  in __sk_msg_free_partial()
244 		sk_msg_free_elem(sk, msg, i, charge);  in __sk_msg_free_partial()
246 		sk_msg_check_to_free(msg, i, bytes);  in __sk_msg_free_partial()
248 	msg->sg.start = i;  in __sk_msg_free_partial()
251 void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes)  in sk_msg_free_partial()  argument
253 	__sk_msg_free_partial(sk, msg, bytes, true);  in sk_msg_free_partial()
257 void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg,  in sk_msg_free_partial_nocharge()  argument
260 	__sk_msg_free_partial(sk, msg, bytes, false);  in sk_msg_free_partial_nocharge()
263 void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len)  in sk_msg_trim()  argument
265 	int trim = msg->sg.size - len;  in sk_msg_trim()
266 	u32 i = msg->sg.end;  in sk_msg_trim()
274 	msg->sg.size = len;  in sk_msg_trim()
275 	while (msg->sg.data[i].length &&  in sk_msg_trim()
276 	       trim >= msg->sg.data[i].length) {  in sk_msg_trim()
277 		trim -= msg->sg.data[i].length;  in sk_msg_trim()
278 		sk_msg_free_elem(sk, msg, i, true);  in sk_msg_trim()
284 	msg->sg.data[i].length -= trim;  in sk_msg_trim()
287 	if (msg->sg.curr == i && msg->sg.copybreak > msg->sg.data[i].length)  in sk_msg_trim()
288 		msg->sg.copybreak = msg->sg.data[i].length;  in sk_msg_trim()
291 	msg->sg.end = i;  in sk_msg_trim()
299 	if (!msg->sg.size) {  in sk_msg_trim()
300 		msg->sg.curr = msg->sg.start;  in sk_msg_trim()
301 		msg->sg.copybreak = 0;  in sk_msg_trim()
302 	} else if (sk_msg_iter_dist(msg->sg.start, msg->sg.curr) >=  in sk_msg_trim()
303 		   sk_msg_iter_dist(msg->sg.start, msg->sg.end)) {  in sk_msg_trim()
305 		msg->sg.curr = i;  in sk_msg_trim()
306 		msg->sg.copybreak = msg->sg.data[i].length;  in sk_msg_trim()
312 			      struct sk_msg *msg, u32 bytes)  in sk_msg_zerocopy_from_iter()  argument
314 	int i, maxpages, ret = 0, num_elems = sk_msg_elem_used(msg);  in sk_msg_zerocopy_from_iter()
319 	orig = msg->sg.size;  in sk_msg_zerocopy_from_iter()
336 		msg->sg.size += copied;  in sk_msg_zerocopy_from_iter()
340 			sg_set_page(&msg->sg.data[msg->sg.end],  in sk_msg_zerocopy_from_iter()
342 			sg_unmark_end(&msg->sg.data[msg->sg.end]);  in sk_msg_zerocopy_from_iter()
347 			sk_msg_iter_next(msg, end);  in sk_msg_zerocopy_from_iter()
355 		msg->sg.copybreak = 0;  in sk_msg_zerocopy_from_iter()
356 		msg->sg.curr = msg->sg.end;  in sk_msg_zerocopy_from_iter()
363 		iov_iter_revert(from, msg->sg.size - orig);  in sk_msg_zerocopy_from_iter()
369 			     struct sk_msg *msg, u32 bytes)  in sk_msg_memcopy_from_iter()  argument
371 	int ret = -ENOSPC, i = msg->sg.curr;  in sk_msg_memcopy_from_iter()
377 		sge = sk_msg_elem(msg, i);  in sk_msg_memcopy_from_iter()
379 		if (msg->sg.copybreak >= sge->length) {  in sk_msg_memcopy_from_iter()
380 			msg->sg.copybreak = 0;  in sk_msg_memcopy_from_iter()
382 			if (i == msg->sg.end)  in sk_msg_memcopy_from_iter()
384 			sge = sk_msg_elem(msg, i);  in sk_msg_memcopy_from_iter()
387 		buf_size = sge->length - msg->sg.copybreak;  in sk_msg_memcopy_from_iter()
389 		to = sg_virt(sge) + msg->sg.copybreak;  in sk_msg_memcopy_from_iter()
390 		msg->sg.copybreak += copy;  in sk_msg_memcopy_from_iter()
402 		msg->sg.copybreak = 0;  in sk_msg_memcopy_from_iter()
404 	} while (i != msg->sg.end);  in sk_msg_memcopy_from_iter()
406 	msg->sg.curr = i;  in sk_msg_memcopy_from_iter()
412 int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,  in sk_msg_recvmsg()  argument
415 	struct iov_iter *iter = &msg->msg_iter;  in sk_msg_recvmsg()
505 	struct sk_msg *msg;  in alloc_sk_msg()  local
507 	msg = kzalloc(sizeof(*msg), gfp | __GFP_NOWARN);  in alloc_sk_msg()
508 	if (unlikely(!msg))  in alloc_sk_msg()
510 	sg_init_marker(msg->sg.data, NR_MSG_FRAG_IDS);  in alloc_sk_msg()
511 	return msg;  in alloc_sk_msg()
530 					struct sk_msg *msg)  in sk_psock_skb_ingress_enqueue()  argument
534 	num_sge = skb_to_sgvec(skb, msg->sg.data, off, len);  in sk_psock_skb_ingress_enqueue()
544 		num_sge = skb_to_sgvec(skb, msg->sg.data, off, len);  in sk_psock_skb_ingress_enqueue()
550 	msg->sg.start = 0;  in sk_psock_skb_ingress_enqueue()
551 	msg->sg.size = copied;  in sk_psock_skb_ingress_enqueue()
552 	msg->sg.end = num_sge;  in sk_psock_skb_ingress_enqueue()
553 	msg->skb = skb;  in sk_psock_skb_ingress_enqueue()
555 	sk_psock_queue_msg(psock, msg);  in sk_psock_skb_ingress_enqueue()
567 	struct sk_msg *msg;  in sk_psock_skb_ingress()  local
576 	msg = sk_psock_create_ingress_msg(sk, skb);  in sk_psock_skb_ingress()
577 	if (!msg)  in sk_psock_skb_ingress()
587 	err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);  in sk_psock_skb_ingress()
589 		kfree(msg);  in sk_psock_skb_ingress()
600 	struct sk_msg *msg = alloc_sk_msg(GFP_ATOMIC);  in sk_psock_skb_ingress_self()  local
604 	if (unlikely(!msg))  in sk_psock_skb_ingress_self()
607 	err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);  in sk_psock_skb_ingress_self()
609 		kfree(msg);  in sk_psock_skb_ingress_self()
771 	struct sk_msg *msg, *tmp;  in __sk_psock_purge_ingress_msg()  local
773 	list_for_each_entry_safe(msg, tmp, &psock->ingress_msg, list) {  in __sk_psock_purge_ingress_msg()
774 		list_del(&msg->list);  in __sk_psock_purge_ingress_msg()
775 		sk_msg_free(psock->sk, msg);  in __sk_psock_purge_ingress_msg()
776 		kfree(msg);  in __sk_psock_purge_ingress_msg()
868 			 struct sk_msg *msg)  in sk_psock_msg_verdict()  argument
880 	sk_msg_compute_data_pointers(msg);  in sk_psock_msg_verdict()
881 	msg->sk = sk;  in sk_psock_msg_verdict()
882 	ret = bpf_prog_run_pin_on_cpu(prog, msg);  in sk_psock_msg_verdict()
883 	ret = sk_psock_map_verd(ret, msg->sk_redir);  in sk_psock_msg_verdict()
884 	psock->apply_bytes = msg->apply_bytes;  in sk_psock_msg_verdict()
890 		if (!msg->sk_redir) {  in sk_psock_msg_verdict()
894 		psock->redir_ingress = sk_msg_to_ingress(msg);  in sk_psock_msg_verdict()
895 		psock->sk_redir = msg->sk_redir;  in sk_psock_msg_verdict()