Lines Matching refs:strp
15 static void tls_strp_abort_strp(struct tls_strparser *strp, int err) in tls_strp_abort_strp() argument
17 if (strp->stopped) in tls_strp_abort_strp()
20 strp->stopped = 1; in tls_strp_abort_strp()
23 WRITE_ONCE(strp->sk->sk_err, -err); in tls_strp_abort_strp()
26 sk_error_report(strp->sk); in tls_strp_abort_strp()
29 static void tls_strp_anchor_free(struct tls_strparser *strp) in tls_strp_anchor_free() argument
31 struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); in tls_strp_anchor_free()
34 if (!strp->copy_mode) in tls_strp_anchor_free()
36 consume_skb(strp->anchor); in tls_strp_anchor_free()
37 strp->anchor = NULL; in tls_strp_anchor_free()
41 tls_strp_skb_copy(struct tls_strparser *strp, struct sk_buff *in_skb, in tls_strp_skb_copy() argument
48 &err, strp->sk->sk_allocation); in tls_strp_skb_copy()
68 static struct sk_buff *tls_strp_msg_make_copy(struct tls_strparser *strp) in tls_strp_msg_make_copy() argument
73 skb = tls_strp_skb_copy(strp, strp->anchor, strp->stm.offset, in tls_strp_msg_make_copy()
74 strp->stm.full_len); in tls_strp_msg_make_copy()
86 struct tls_strparser *strp = &ctx->strp; in tls_strp_msg_detach() local
89 DEBUG_NET_WARN_ON_ONCE(!strp->anchor->decrypted); in tls_strp_msg_detach()
97 if (strp->copy_mode) { in tls_strp_msg_detach()
104 skb = alloc_skb(0, strp->sk->sk_allocation); in tls_strp_msg_detach()
108 swap(strp->anchor, skb); in tls_strp_msg_detach()
112 return tls_strp_msg_make_copy(strp); in tls_strp_msg_detach()
121 struct tls_strparser *strp = &ctx->strp; in tls_strp_msg_cow() local
124 if (strp->copy_mode) in tls_strp_msg_cow()
127 skb = tls_strp_msg_make_copy(strp); in tls_strp_msg_cow()
131 tls_strp_anchor_free(strp); in tls_strp_msg_cow()
132 strp->anchor = skb; in tls_strp_msg_cow()
134 tcp_read_done(strp->sk, strp->stm.full_len); in tls_strp_msg_cow()
135 strp->copy_mode = 1; in tls_strp_msg_cow()
144 int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst) in tls_strp_msg_hold() argument
146 struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); in tls_strp_msg_hold()
148 if (strp->copy_mode) { in tls_strp_msg_hold()
154 skb = alloc_skb(0, strp->sk->sk_allocation); in tls_strp_msg_hold()
158 __skb_queue_tail(dst, strp->anchor); in tls_strp_msg_hold()
159 strp->anchor = skb; in tls_strp_msg_hold()
164 offset = strp->stm.offset; in tls_strp_msg_hold()
165 len = strp->stm.full_len; in tls_strp_msg_hold()
177 clone = skb_clone(iter, strp->sk->sk_allocation); in tls_strp_msg_hold()
191 static void tls_strp_flush_anchor_copy(struct tls_strparser *strp) in tls_strp_flush_anchor_copy() argument
193 struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); in tls_strp_flush_anchor_copy()
201 if (strp->copy_mode) { in tls_strp_flush_anchor_copy()
205 strp->copy_mode = 0; in tls_strp_flush_anchor_copy()
206 strp->mixed_decrypted = 0; in tls_strp_flush_anchor_copy()
209 static int tls_strp_copyin_frag(struct tls_strparser *strp, struct sk_buff *skb, in tls_strp_copyin_frag() argument
221 if (!strp->stm.full_len) { in tls_strp_copyin_frag()
233 sz = tls_rx_msg_size(strp, skb); in tls_strp_copyin_frag()
253 strp->stm.full_len = sz; in tls_strp_copyin_frag()
254 if (!strp->stm.full_len) in tls_strp_copyin_frag()
259 while (len && strp->stm.full_len > skb->len) { in tls_strp_copyin_frag()
260 chunk = min_t(size_t, len, strp->stm.full_len - skb->len); in tls_strp_copyin_frag()
279 static int tls_strp_copyin_skb(struct tls_strparser *strp, struct sk_buff *skb, in tls_strp_copyin_skb() argument
288 if (strp->stm.full_len) in tls_strp_copyin_skb()
289 chunk = strp->stm.full_len - skb->len; in tls_strp_copyin_skb()
294 nskb = tls_strp_skb_copy(strp, in_skb, offset, chunk); in tls_strp_copyin_skb()
312 if (!strp->stm.full_len) { in tls_strp_copyin_skb()
313 sz = tls_rx_msg_size(strp, skb); in tls_strp_copyin_skb()
329 strp->stm.full_len = sz; in tls_strp_copyin_skb()
338 struct tls_strparser *strp = (struct tls_strparser *)desc->arg.data; in tls_strp_copyin() local
342 if (strp->msg_ready) in tls_strp_copyin()
345 skb = strp->anchor; in tls_strp_copyin()
349 strp->mixed_decrypted |= !!skb_cmp_decrypted(skb, in_skb); in tls_strp_copyin()
351 if (IS_ENABLED(CONFIG_TLS_DEVICE) && strp->mixed_decrypted) in tls_strp_copyin()
352 ret = tls_strp_copyin_skb(strp, skb, in_skb, offset, in_len); in tls_strp_copyin()
354 ret = tls_strp_copyin_frag(strp, skb, in_skb, offset, in_len); in tls_strp_copyin()
360 if (strp->stm.full_len && strp->stm.full_len == skb->len) { in tls_strp_copyin()
363 WRITE_ONCE(strp->msg_ready, 1); in tls_strp_copyin()
364 tls_rx_msg_ready(strp); in tls_strp_copyin()
370 static int tls_strp_read_copyin(struct tls_strparser *strp) in tls_strp_read_copyin() argument
374 desc.arg.data = strp; in tls_strp_read_copyin()
379 tcp_read_sock(strp->sk, &desc, tls_strp_copyin); in tls_strp_read_copyin()
384 static int tls_strp_read_copy(struct tls_strparser *strp, bool qshort) in tls_strp_read_copy() argument
394 if (likely(qshort && !tcp_epollin_ready(strp->sk, INT_MAX))) in tls_strp_read_copy()
397 shinfo = skb_shinfo(strp->anchor); in tls_strp_read_copy()
401 need_spc = strp->stm.full_len ?: TLS_MAX_PAYLOAD_SIZE + PAGE_SIZE; in tls_strp_read_copy()
404 page = alloc_page(strp->sk->sk_allocation); in tls_strp_read_copy()
406 tls_strp_flush_anchor_copy(strp); in tls_strp_read_copy()
410 skb_fill_page_desc(strp->anchor, shinfo->nr_frags++, in tls_strp_read_copy()
414 strp->copy_mode = 1; in tls_strp_read_copy()
415 strp->stm.offset = 0; in tls_strp_read_copy()
417 strp->anchor->len = 0; in tls_strp_read_copy()
418 strp->anchor->data_len = 0; in tls_strp_read_copy()
419 strp->anchor->truesize = round_up(need_spc, PAGE_SIZE); in tls_strp_read_copy()
421 tls_strp_read_copyin(strp); in tls_strp_read_copy()
426 static bool tls_strp_check_queue_ok(struct tls_strparser *strp) in tls_strp_check_queue_ok() argument
428 unsigned int len = strp->stm.offset + strp->stm.full_len; in tls_strp_check_queue_ok()
432 first = skb_shinfo(strp->anchor)->frag_list; in tls_strp_check_queue_ok()
453 static void tls_strp_load_anchor_with_queue(struct tls_strparser *strp, int len) in tls_strp_load_anchor_with_queue() argument
455 struct tcp_sock *tp = tcp_sk(strp->sk); in tls_strp_load_anchor_with_queue()
459 first = tcp_recv_skb(strp->sk, tp->copied_seq, &offset); in tls_strp_load_anchor_with_queue()
464 strp->anchor->len = offset + len; in tls_strp_load_anchor_with_queue()
465 strp->anchor->data_len = offset + len; in tls_strp_load_anchor_with_queue()
466 strp->anchor->truesize = offset + len; in tls_strp_load_anchor_with_queue()
468 skb_shinfo(strp->anchor)->frag_list = first; in tls_strp_load_anchor_with_queue()
470 skb_copy_header(strp->anchor, first); in tls_strp_load_anchor_with_queue()
471 strp->anchor->destructor = NULL; in tls_strp_load_anchor_with_queue()
473 strp->stm.offset = offset; in tls_strp_load_anchor_with_queue()
476 void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh) in tls_strp_msg_load() argument
481 DEBUG_NET_WARN_ON_ONCE(!strp->msg_ready); in tls_strp_msg_load()
482 DEBUG_NET_WARN_ON_ONCE(!strp->stm.full_len); in tls_strp_msg_load()
484 if (!strp->copy_mode && force_refresh) { in tls_strp_msg_load()
485 if (WARN_ON(tcp_inq(strp->sk) < strp->stm.full_len)) in tls_strp_msg_load()
488 tls_strp_load_anchor_with_queue(strp, strp->stm.full_len); in tls_strp_msg_load()
491 rxm = strp_msg(strp->anchor); in tls_strp_msg_load()
492 rxm->full_len = strp->stm.full_len; in tls_strp_msg_load()
493 rxm->offset = strp->stm.offset; in tls_strp_msg_load()
494 tlm = tls_msg(strp->anchor); in tls_strp_msg_load()
495 tlm->control = strp->mark; in tls_strp_msg_load()
499 static int tls_strp_read_sock(struct tls_strparser *strp) in tls_strp_read_sock() argument
503 inq = tcp_inq(strp->sk); in tls_strp_read_sock()
507 if (unlikely(strp->copy_mode)) in tls_strp_read_sock()
508 return tls_strp_read_copyin(strp); in tls_strp_read_sock()
510 if (inq < strp->stm.full_len) in tls_strp_read_sock()
511 return tls_strp_read_copy(strp, true); in tls_strp_read_sock()
513 if (!strp->stm.full_len) { in tls_strp_read_sock()
514 tls_strp_load_anchor_with_queue(strp, inq); in tls_strp_read_sock()
516 sz = tls_rx_msg_size(strp, strp->anchor); in tls_strp_read_sock()
518 tls_strp_abort_strp(strp, sz); in tls_strp_read_sock()
522 strp->stm.full_len = sz; in tls_strp_read_sock()
524 if (!strp->stm.full_len || inq < strp->stm.full_len) in tls_strp_read_sock()
525 return tls_strp_read_copy(strp, true); in tls_strp_read_sock()
528 if (!tls_strp_check_queue_ok(strp)) in tls_strp_read_sock()
529 return tls_strp_read_copy(strp, false); in tls_strp_read_sock()
531 WRITE_ONCE(strp->msg_ready, 1); in tls_strp_read_sock()
532 tls_rx_msg_ready(strp); in tls_strp_read_sock()
537 void tls_strp_check_rcv(struct tls_strparser *strp) in tls_strp_check_rcv() argument
539 if (unlikely(strp->stopped) || strp->msg_ready) in tls_strp_check_rcv()
542 if (tls_strp_read_sock(strp) == -ENOMEM) in tls_strp_check_rcv()
543 queue_work(tls_strp_wq, &strp->work); in tls_strp_check_rcv()
547 void tls_strp_data_ready(struct tls_strparser *strp) in tls_strp_data_ready() argument
556 if (sock_owned_by_user_nocheck(strp->sk)) { in tls_strp_data_ready()
557 queue_work(tls_strp_wq, &strp->work); in tls_strp_data_ready()
561 tls_strp_check_rcv(strp); in tls_strp_data_ready()
566 struct tls_strparser *strp = in tls_strp_work() local
569 lock_sock(strp->sk); in tls_strp_work()
570 tls_strp_check_rcv(strp); in tls_strp_work()
571 release_sock(strp->sk); in tls_strp_work()
574 void tls_strp_msg_done(struct tls_strparser *strp) in tls_strp_msg_done() argument
576 WARN_ON(!strp->stm.full_len); in tls_strp_msg_done()
578 if (likely(!strp->copy_mode)) in tls_strp_msg_done()
579 tcp_read_done(strp->sk, strp->stm.full_len); in tls_strp_msg_done()
581 tls_strp_flush_anchor_copy(strp); in tls_strp_msg_done()
583 WRITE_ONCE(strp->msg_ready, 0); in tls_strp_msg_done()
584 memset(&strp->stm, 0, sizeof(strp->stm)); in tls_strp_msg_done()
586 tls_strp_check_rcv(strp); in tls_strp_msg_done()
589 void tls_strp_stop(struct tls_strparser *strp) in tls_strp_stop() argument
591 strp->stopped = 1; in tls_strp_stop()
594 int tls_strp_init(struct tls_strparser *strp, struct sock *sk) in tls_strp_init() argument
596 memset(strp, 0, sizeof(*strp)); in tls_strp_init()
598 strp->sk = sk; in tls_strp_init()
600 strp->anchor = alloc_skb(0, GFP_KERNEL); in tls_strp_init()
601 if (!strp->anchor) in tls_strp_init()
604 INIT_WORK(&strp->work, tls_strp_work); in tls_strp_init()
612 void tls_strp_done(struct tls_strparser *strp) in tls_strp_done() argument
614 WARN_ON(!strp->stopped); in tls_strp_done()
616 cancel_work_sync(&strp->work); in tls_strp_done()
617 tls_strp_anchor_free(strp); in tls_strp_done()