1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 22100c8d2SYuchung Cheng #include <linux/kernel.h> 310467163SJerry Chu #include <linux/tcp.h> 410467163SJerry Chu #include <linux/rcupdate.h> 510467163SJerry Chu #include <net/tcp.h> 62100c8d2SYuchung Cheng 743713848SHaishuang Yan void tcp_fastopen_init_key_once(struct net *net) 8222e83d2SHannes Frederic Sowa { 943713848SHaishuang Yan u8 key[TCP_FASTOPEN_KEY_LENGTH]; 1043713848SHaishuang Yan struct tcp_fastopen_context *ctxt; 1143713848SHaishuang Yan 1243713848SHaishuang Yan rcu_read_lock(); 1343713848SHaishuang Yan ctxt = rcu_dereference(net->ipv4.tcp_fastopen_ctx); 1443713848SHaishuang Yan if (ctxt) { 1543713848SHaishuang Yan rcu_read_unlock(); 1643713848SHaishuang Yan return; 1743713848SHaishuang Yan } 1843713848SHaishuang Yan rcu_read_unlock(); 19222e83d2SHannes Frederic Sowa 20222e83d2SHannes Frederic Sowa /* tcp_fastopen_reset_cipher publishes the new context 21222e83d2SHannes Frederic Sowa * atomically, so we allow this race happening here. 22222e83d2SHannes Frederic Sowa * 23222e83d2SHannes Frederic Sowa * All call sites of tcp_fastopen_cookie_gen also check 24222e83d2SHannes Frederic Sowa * for a valid cookie, so this is an acceptable risk. 25222e83d2SHannes Frederic Sowa */ 2643713848SHaishuang Yan get_random_bytes(key, sizeof(key)); 27438ac880SArd Biesheuvel tcp_fastopen_reset_cipher(net, NULL, key, NULL); 28222e83d2SHannes Frederic Sowa } 29222e83d2SHannes Frederic Sowa 3010467163SJerry Chu static void tcp_fastopen_ctx_free(struct rcu_head *head) 3110467163SJerry Chu { 3210467163SJerry Chu struct tcp_fastopen_context *ctx = 3310467163SJerry Chu container_of(head, struct tcp_fastopen_context, rcu); 349092a76dSJason Baron 35453431a5SWaiman Long kfree_sensitive(ctx); 3610467163SJerry Chu } 3710467163SJerry Chu 381fba70e5SYuchung Cheng void tcp_fastopen_destroy_cipher(struct sock *sk) 391fba70e5SYuchung Cheng { 401fba70e5SYuchung Cheng struct tcp_fastopen_context *ctx; 411fba70e5SYuchung Cheng 421fba70e5SYuchung Cheng ctx = rcu_dereference_protected( 431fba70e5SYuchung Cheng inet_csk(sk)->icsk_accept_queue.fastopenq.ctx, 1); 441fba70e5SYuchung Cheng if (ctx) 451fba70e5SYuchung Cheng call_rcu(&ctx->rcu, tcp_fastopen_ctx_free); 461fba70e5SYuchung Cheng } 471fba70e5SYuchung Cheng 4843713848SHaishuang Yan void tcp_fastopen_ctx_destroy(struct net *net) 4943713848SHaishuang Yan { 5043713848SHaishuang Yan struct tcp_fastopen_context *ctxt; 5143713848SHaishuang Yan 52e93abb84SEric Dumazet ctxt = xchg((__force struct tcp_fastopen_context **)&net->ipv4.tcp_fastopen_ctx, NULL); 5343713848SHaishuang Yan 5443713848SHaishuang Yan if (ctxt) 5543713848SHaishuang Yan call_rcu(&ctxt->rcu, tcp_fastopen_ctx_free); 5643713848SHaishuang Yan } 5743713848SHaishuang Yan 581fba70e5SYuchung Cheng int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk, 59438ac880SArd Biesheuvel void *primary_key, void *backup_key) 6010467163SJerry Chu { 6110467163SJerry Chu struct tcp_fastopen_context *ctx, *octx; 621fba70e5SYuchung Cheng struct fastopen_queue *q; 639092a76dSJason Baron int err = 0; 6410467163SJerry Chu 65c681edaeSArd Biesheuvel ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); 66c681edaeSArd Biesheuvel if (!ctx) { 67c681edaeSArd Biesheuvel err = -ENOMEM; 689092a76dSJason Baron goto out; 6910467163SJerry Chu } 70c681edaeSArd Biesheuvel 71438ac880SArd Biesheuvel ctx->key[0].key[0] = get_unaligned_le64(primary_key); 72438ac880SArd Biesheuvel ctx->key[0].key[1] = get_unaligned_le64(primary_key + 8); 73c681edaeSArd Biesheuvel if (backup_key) { 74438ac880SArd Biesheuvel ctx->key[1].key[0] = get_unaligned_le64(backup_key); 75438ac880SArd Biesheuvel ctx->key[1].key[1] = get_unaligned_le64(backup_key + 8); 76c681edaeSArd Biesheuvel ctx->num = 2; 77c681edaeSArd Biesheuvel } else { 78c681edaeSArd Biesheuvel ctx->num = 1; 79c681edaeSArd Biesheuvel } 80c681edaeSArd Biesheuvel 811fba70e5SYuchung Cheng if (sk) { 821fba70e5SYuchung Cheng q = &inet_csk(sk)->icsk_accept_queue.fastopenq; 83e93abb84SEric Dumazet octx = xchg((__force struct tcp_fastopen_context **)&q->ctx, ctx); 841fba70e5SYuchung Cheng } else { 85e93abb84SEric Dumazet octx = xchg((__force struct tcp_fastopen_context **)&net->ipv4.tcp_fastopen_ctx, ctx); 861fba70e5SYuchung Cheng } 8710467163SJerry Chu 8810467163SJerry Chu if (octx) 8910467163SJerry Chu call_rcu(&octx->rcu, tcp_fastopen_ctx_free); 909092a76dSJason Baron out: 9110467163SJerry Chu return err; 9210467163SJerry Chu } 9310467163SJerry Chu 94f19008e6SJason Baron int tcp_fastopen_get_cipher(struct net *net, struct inet_connection_sock *icsk, 95f19008e6SJason Baron u64 *key) 96f19008e6SJason Baron { 97f19008e6SJason Baron struct tcp_fastopen_context *ctx; 98f19008e6SJason Baron int n_keys = 0, i; 99f19008e6SJason Baron 100f19008e6SJason Baron rcu_read_lock(); 101f19008e6SJason Baron if (icsk) 102f19008e6SJason Baron ctx = rcu_dereference(icsk->icsk_accept_queue.fastopenq.ctx); 103f19008e6SJason Baron else 104f19008e6SJason Baron ctx = rcu_dereference(net->ipv4.tcp_fastopen_ctx); 105f19008e6SJason Baron if (ctx) { 106f19008e6SJason Baron n_keys = tcp_fastopen_context_len(ctx); 107f19008e6SJason Baron for (i = 0; i < n_keys; i++) { 108f19008e6SJason Baron put_unaligned_le64(ctx->key[i].key[0], key + (i * 2)); 109f19008e6SJason Baron put_unaligned_le64(ctx->key[i].key[1], key + (i * 2) + 1); 110f19008e6SJason Baron } 111f19008e6SJason Baron } 112f19008e6SJason Baron rcu_read_unlock(); 113f19008e6SJason Baron 114f19008e6SJason Baron return n_keys; 115f19008e6SJason Baron } 116f19008e6SJason Baron 117483642e5SChristoph Paasch static bool __tcp_fastopen_cookie_gen_cipher(struct request_sock *req, 118483642e5SChristoph Paasch struct sk_buff *syn, 119438ac880SArd Biesheuvel const siphash_key_t *key, 120149479d0SYuchung Cheng struct tcp_fastopen_cookie *foc) 12110467163SJerry Chu { 122c681edaeSArd Biesheuvel BUILD_BUG_ON(TCP_FASTOPEN_COOKIE_SIZE != sizeof(u64)); 123c681edaeSArd Biesheuvel 124483642e5SChristoph Paasch if (req->rsk_ops->family == AF_INET) { 125483642e5SChristoph Paasch const struct iphdr *iph = ip_hdr(syn); 12610467163SJerry Chu 127438ac880SArd Biesheuvel foc->val[0] = cpu_to_le64(siphash(&iph->saddr, 128c681edaeSArd Biesheuvel sizeof(iph->saddr) + 129c681edaeSArd Biesheuvel sizeof(iph->daddr), 130438ac880SArd Biesheuvel key)); 13110467163SJerry Chu foc->len = TCP_FASTOPEN_COOKIE_SIZE; 132483642e5SChristoph Paasch return true; 13310467163SJerry Chu } 134483642e5SChristoph Paasch #if IS_ENABLED(CONFIG_IPV6) 135483642e5SChristoph Paasch if (req->rsk_ops->family == AF_INET6) { 136483642e5SChristoph Paasch const struct ipv6hdr *ip6h = ipv6_hdr(syn); 137483642e5SChristoph Paasch 138438ac880SArd Biesheuvel foc->val[0] = cpu_to_le64(siphash(&ip6h->saddr, 139c681edaeSArd Biesheuvel sizeof(ip6h->saddr) + 140c681edaeSArd Biesheuvel sizeof(ip6h->daddr), 141438ac880SArd Biesheuvel key)); 142483642e5SChristoph Paasch foc->len = TCP_FASTOPEN_COOKIE_SIZE; 143483642e5SChristoph Paasch return true; 144483642e5SChristoph Paasch } 145483642e5SChristoph Paasch #endif 146483642e5SChristoph Paasch return false; 1473a19ce0eSDaniel Lee } 1483a19ce0eSDaniel Lee 149c681edaeSArd Biesheuvel /* Generate the fastopen cookie by applying SipHash to both the source and 150c681edaeSArd Biesheuvel * destination addresses. 1513a19ce0eSDaniel Lee */ 1529092a76dSJason Baron static void tcp_fastopen_cookie_gen(struct sock *sk, 15343713848SHaishuang Yan struct request_sock *req, 1543a19ce0eSDaniel Lee struct sk_buff *syn, 1553a19ce0eSDaniel Lee struct tcp_fastopen_cookie *foc) 1563a19ce0eSDaniel Lee { 157483642e5SChristoph Paasch struct tcp_fastopen_context *ctx; 1583a19ce0eSDaniel Lee 159483642e5SChristoph Paasch rcu_read_lock(); 1609092a76dSJason Baron ctx = tcp_fastopen_get_ctx(sk); 161483642e5SChristoph Paasch if (ctx) 162438ac880SArd Biesheuvel __tcp_fastopen_cookie_gen_cipher(req, syn, &ctx->key[0], foc); 163483642e5SChristoph Paasch rcu_read_unlock(); 16410467163SJerry Chu } 1655b7ed089SYuchung Cheng 16661d2bcaeSEric Dumazet /* If an incoming SYN or SYNACK frame contains a payload and/or FIN, 16761d2bcaeSEric Dumazet * queue this additional data / FIN. 16861d2bcaeSEric Dumazet */ 16961d2bcaeSEric Dumazet void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb) 17061d2bcaeSEric Dumazet { 17161d2bcaeSEric Dumazet struct tcp_sock *tp = tcp_sk(sk); 17261d2bcaeSEric Dumazet 17361d2bcaeSEric Dumazet if (TCP_SKB_CB(skb)->end_seq == tp->rcv_nxt) 17461d2bcaeSEric Dumazet return; 17561d2bcaeSEric Dumazet 17661d2bcaeSEric Dumazet skb = skb_clone(skb, GFP_ATOMIC); 17761d2bcaeSEric Dumazet if (!skb) 17861d2bcaeSEric Dumazet return; 17961d2bcaeSEric Dumazet 18061d2bcaeSEric Dumazet skb_dst_drop(skb); 181a44d6eacSMartin KaFai Lau /* segs_in has been initialized to 1 in tcp_create_openreq_child(). 182a44d6eacSMartin KaFai Lau * Hence, reset segs_in to 0 before calling tcp_segs_in() 183a44d6eacSMartin KaFai Lau * to avoid double counting. Also, tcp_segs_in() expects 184a44d6eacSMartin KaFai Lau * skb->len to include the tcp_hdrlen. Hence, it should 185a44d6eacSMartin KaFai Lau * be called before __skb_pull(). 186a44d6eacSMartin KaFai Lau */ 187a44d6eacSMartin KaFai Lau tp->segs_in = 0; 188a44d6eacSMartin KaFai Lau tcp_segs_in(tp, skb); 18961d2bcaeSEric Dumazet __skb_pull(skb, tcp_hdrlen(skb)); 19076061f63SEric Dumazet sk_forced_mem_schedule(sk, skb->truesize); 19161d2bcaeSEric Dumazet skb_set_owner_r(skb, sk); 19261d2bcaeSEric Dumazet 1939d691539SEric Dumazet TCP_SKB_CB(skb)->seq++; 1949d691539SEric Dumazet TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_SYN; 1959d691539SEric Dumazet 19661d2bcaeSEric Dumazet tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq; 19761d2bcaeSEric Dumazet __skb_queue_tail(&sk->sk_receive_queue, skb); 19861d2bcaeSEric Dumazet tp->syn_data_acked = 1; 19961d2bcaeSEric Dumazet 20061d2bcaeSEric Dumazet /* u64_stats_update_begin(&tp->syncp) not needed here, 20161d2bcaeSEric Dumazet * as we certainly are not changing upper 32bit value (0) 20261d2bcaeSEric Dumazet */ 20361d2bcaeSEric Dumazet tp->bytes_received = skb->len; 204e3e17b77SEric Dumazet 205e3e17b77SEric Dumazet if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 206e3e17b77SEric Dumazet tcp_fin(sk); 20761d2bcaeSEric Dumazet } 20861d2bcaeSEric Dumazet 2099092a76dSJason Baron /* returns 0 - no key match, 1 for primary, 2 for backup */ 2109092a76dSJason Baron static int tcp_fastopen_cookie_gen_check(struct sock *sk, 2119092a76dSJason Baron struct request_sock *req, 2129092a76dSJason Baron struct sk_buff *syn, 2139092a76dSJason Baron struct tcp_fastopen_cookie *orig, 2149092a76dSJason Baron struct tcp_fastopen_cookie *valid_foc) 2159092a76dSJason Baron { 2169092a76dSJason Baron struct tcp_fastopen_cookie search_foc = { .len = -1 }; 2179092a76dSJason Baron struct tcp_fastopen_cookie *foc = valid_foc; 2189092a76dSJason Baron struct tcp_fastopen_context *ctx; 2199092a76dSJason Baron int i, ret = 0; 2209092a76dSJason Baron 2219092a76dSJason Baron rcu_read_lock(); 2229092a76dSJason Baron ctx = tcp_fastopen_get_ctx(sk); 2239092a76dSJason Baron if (!ctx) 2249092a76dSJason Baron goto out; 2259092a76dSJason Baron for (i = 0; i < tcp_fastopen_context_len(ctx); i++) { 226438ac880SArd Biesheuvel __tcp_fastopen_cookie_gen_cipher(req, syn, &ctx->key[i], foc); 2279092a76dSJason Baron if (tcp_fastopen_cookie_match(foc, orig)) { 2289092a76dSJason Baron ret = i + 1; 2299092a76dSJason Baron goto out; 2309092a76dSJason Baron } 2319092a76dSJason Baron foc = &search_foc; 2329092a76dSJason Baron } 2339092a76dSJason Baron out: 2349092a76dSJason Baron rcu_read_unlock(); 2359092a76dSJason Baron return ret; 2369092a76dSJason Baron } 2379092a76dSJason Baron 2387c85af88SEric Dumazet static struct sock *tcp_fastopen_create_child(struct sock *sk, 2395b7ed089SYuchung Cheng struct sk_buff *skb, 2405b7ed089SYuchung Cheng struct request_sock *req) 2415b7ed089SYuchung Cheng { 24217846376SDave Jones struct tcp_sock *tp; 2435b7ed089SYuchung Cheng struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue; 2445b7ed089SYuchung Cheng struct sock *child; 2455e0724d0SEric Dumazet bool own_req; 2465b7ed089SYuchung Cheng 2475e0724d0SEric Dumazet child = inet_csk(sk)->icsk_af_ops->syn_recv_sock(sk, skb, req, NULL, 2485e0724d0SEric Dumazet NULL, &own_req); 24951456b29SIan Morris if (!child) 2507c85af88SEric Dumazet return NULL; 2515b7ed089SYuchung Cheng 2520536fcc0SEric Dumazet spin_lock(&queue->fastopenq.lock); 2530536fcc0SEric Dumazet queue->fastopenq.qlen++; 2540536fcc0SEric Dumazet spin_unlock(&queue->fastopenq.lock); 2555b7ed089SYuchung Cheng 2565b7ed089SYuchung Cheng /* Initialize the child socket. Have to fix some values to take 2575b7ed089SYuchung Cheng * into account the child is a Fast Open socket and is created 2585b7ed089SYuchung Cheng * only out of the bits carried in the SYN packet. 2595b7ed089SYuchung Cheng */ 2605b7ed089SYuchung Cheng tp = tcp_sk(child); 2615b7ed089SYuchung Cheng 262d983ea6fSEric Dumazet rcu_assign_pointer(tp->fastopen_rsk, req); 2639439ce00SEric Dumazet tcp_rsk(req)->tfo_listener = true; 2645b7ed089SYuchung Cheng 2655b7ed089SYuchung Cheng /* RFC1323: The window in SYN & SYN/ACK segments is never 2665b7ed089SYuchung Cheng * scaled. So correct it appropriately. 2675b7ed089SYuchung Cheng */ 2685b7ed089SYuchung Cheng tp->snd_wnd = ntohs(tcp_hdr(skb)->window); 2690dbd7ff3SAlexey Kodanev tp->max_window = tp->snd_wnd; 2705b7ed089SYuchung Cheng 2715b7ed089SYuchung Cheng /* Activate the retrans timer so that SYNACK can be retransmitted. 272ca6fb065SEric Dumazet * The request socket is not added to the ehash 2735b7ed089SYuchung Cheng * because it's been added to the accept queue directly. 2745b7ed089SYuchung Cheng */ 2758ea731d4SJie Meng req->timeout = tcp_timeout_init(child); 2765b7ed089SYuchung Cheng inet_csk_reset_xmit_timer(child, ICSK_TIME_RETRANS, 2778ea731d4SJie Meng req->timeout, TCP_RTO_MAX); 2785b7ed089SYuchung Cheng 27941c6d650SReshetova, Elena refcount_set(&req->rsk_refcnt, 2); 2805b7ed089SYuchung Cheng 2815b7ed089SYuchung Cheng /* Now finish processing the fastopen child socket. */ 28272be0fe6SMartin KaFai Lau tcp_init_transfer(child, BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB, skb); 2835b7ed089SYuchung Cheng 28461d2bcaeSEric Dumazet tp->rcv_nxt = TCP_SKB_CB(skb)->seq + 1; 285ba34e6d9SEric Dumazet 28661d2bcaeSEric Dumazet tcp_fastopen_add_skb(child, skb); 287d654976cSEric Dumazet 28861d2bcaeSEric Dumazet tcp_rsk(req)->rcv_nxt = tp->rcv_nxt; 28928b346cbSNeal Cardwell tp->rcv_wup = tp->rcv_nxt; 2907656d842SEric Dumazet /* tcp_conn_request() is sending the SYNACK, 2917656d842SEric Dumazet * and queues the child into listener accept queue. 2927c85af88SEric Dumazet */ 2937c85af88SEric Dumazet return child; 2945b7ed089SYuchung Cheng } 2955b7ed089SYuchung Cheng 2965b7ed089SYuchung Cheng static bool tcp_fastopen_queue_check(struct sock *sk) 2975b7ed089SYuchung Cheng { 2985b7ed089SYuchung Cheng struct fastopen_queue *fastopenq; 299*70f360ddSEric Dumazet int max_qlen; 3005b7ed089SYuchung Cheng 3015b7ed089SYuchung Cheng /* Make sure the listener has enabled fastopen, and we don't 3025b7ed089SYuchung Cheng * exceed the max # of pending TFO requests allowed before trying 3035b7ed089SYuchung Cheng * to validating the cookie in order to avoid burning CPU cycles 3045b7ed089SYuchung Cheng * unnecessarily. 3055b7ed089SYuchung Cheng * 3065b7ed089SYuchung Cheng * XXX (TFO) - The implication of checking the max_qlen before 3075b7ed089SYuchung Cheng * processing a cookie request is that clients can't differentiate 3085b7ed089SYuchung Cheng * between qlen overflow causing Fast Open to be disabled 3095b7ed089SYuchung Cheng * temporarily vs a server not supporting Fast Open at all. 3105b7ed089SYuchung Cheng */ 3110536fcc0SEric Dumazet fastopenq = &inet_csk(sk)->icsk_accept_queue.fastopenq; 312*70f360ddSEric Dumazet max_qlen = READ_ONCE(fastopenq->max_qlen); 313*70f360ddSEric Dumazet if (max_qlen == 0) 3145b7ed089SYuchung Cheng return false; 3155b7ed089SYuchung Cheng 316*70f360ddSEric Dumazet if (fastopenq->qlen >= max_qlen) { 3175b7ed089SYuchung Cheng struct request_sock *req1; 3185b7ed089SYuchung Cheng spin_lock(&fastopenq->lock); 3195b7ed089SYuchung Cheng req1 = fastopenq->rskq_rst_head; 320fa76ce73SEric Dumazet if (!req1 || time_after(req1->rsk_timer.expires, jiffies)) { 32102a1d6e7SEric Dumazet __NET_INC_STATS(sock_net(sk), 3225b7ed089SYuchung Cheng LINUX_MIB_TCPFASTOPENLISTENOVERFLOW); 323c10d9310SEric Dumazet spin_unlock(&fastopenq->lock); 3245b7ed089SYuchung Cheng return false; 3255b7ed089SYuchung Cheng } 3265b7ed089SYuchung Cheng fastopenq->rskq_rst_head = req1->dl_next; 3275b7ed089SYuchung Cheng fastopenq->qlen--; 3285b7ed089SYuchung Cheng spin_unlock(&fastopenq->lock); 32913854e5aSEric Dumazet reqsk_put(req1); 3305b7ed089SYuchung Cheng } 3315b7ed089SYuchung Cheng return true; 3325b7ed089SYuchung Cheng } 3335b7ed089SYuchung Cheng 33471c02379SChristoph Paasch static bool tcp_fastopen_no_cookie(const struct sock *sk, 33571c02379SChristoph Paasch const struct dst_entry *dst, 33671c02379SChristoph Paasch int flag) 33771c02379SChristoph Paasch { 3385a542133SKuniyuki Iwashima return (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen) & flag) || 33971c02379SChristoph Paasch tcp_sk(sk)->fastopen_no_cookie || 34071c02379SChristoph Paasch (dst && dst_metric(dst, RTAX_FASTOPEN_NO_COOKIE)); 34171c02379SChristoph Paasch } 34271c02379SChristoph Paasch 34389278c9dSYuchung Cheng /* Returns true if we should perform Fast Open on the SYN. The cookie (foc) 34489278c9dSYuchung Cheng * may be updated and return the client in the SYN-ACK later. E.g., Fast Open 34589278c9dSYuchung Cheng * cookie request (foc->len == 0). 34689278c9dSYuchung Cheng */ 3477c85af88SEric Dumazet struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb, 3485b7ed089SYuchung Cheng struct request_sock *req, 34971c02379SChristoph Paasch struct tcp_fastopen_cookie *foc, 35071c02379SChristoph Paasch const struct dst_entry *dst) 3515b7ed089SYuchung Cheng { 35289278c9dSYuchung Cheng bool syn_data = TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq + 1; 3535a542133SKuniyuki Iwashima int tcp_fastopen = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen); 354e1cfcbe8SHaishuang Yan struct tcp_fastopen_cookie valid_foc = { .len = -1 }; 3557c85af88SEric Dumazet struct sock *child; 3569092a76dSJason Baron int ret = 0; 3575b7ed089SYuchung Cheng 358531c94a9SYuchung Cheng if (foc->len == 0) /* Client requests a cookie */ 359c10d9310SEric Dumazet NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFASTOPENCOOKIEREQD); 360531c94a9SYuchung Cheng 361e1cfcbe8SHaishuang Yan if (!((tcp_fastopen & TFO_SERVER_ENABLE) && 36289278c9dSYuchung Cheng (syn_data || foc->len >= 0) && 36389278c9dSYuchung Cheng tcp_fastopen_queue_check(sk))) { 36489278c9dSYuchung Cheng foc->len = -1; 3657c85af88SEric Dumazet return NULL; 3665b7ed089SYuchung Cheng } 36789278c9dSYuchung Cheng 368e3faa49bSLuke Hsiao if (tcp_fastopen_no_cookie(sk, dst, TFO_SERVER_COOKIE_NOT_REQD)) 36989278c9dSYuchung Cheng goto fastopen; 37089278c9dSYuchung Cheng 3719092a76dSJason Baron if (foc->len == 0) { 3729092a76dSJason Baron /* Client requests a cookie. */ 3739092a76dSJason Baron tcp_fastopen_cookie_gen(sk, req, skb, &valid_foc); 3749092a76dSJason Baron } else if (foc->len > 0) { 3759092a76dSJason Baron ret = tcp_fastopen_cookie_gen_check(sk, req, skb, foc, 3769092a76dSJason Baron &valid_foc); 3779092a76dSJason Baron if (!ret) { 3789092a76dSJason Baron NET_INC_STATS(sock_net(sk), 3799092a76dSJason Baron LINUX_MIB_TCPFASTOPENPASSIVEFAIL); 3809092a76dSJason Baron } else { 3819092a76dSJason Baron /* Cookie is valid. Create a (full) child socket to 3829092a76dSJason Baron * accept the data in SYN before returning a SYN-ACK to 3839092a76dSJason Baron * ack the data. If we fail to create the socket, fall 3849092a76dSJason Baron * back and ack the ISN only but includes the same 3859092a76dSJason Baron * cookie. 386843f4a55SYuchung Cheng * 3879092a76dSJason Baron * Note: Data-less SYN with valid cookie is allowed to 3889092a76dSJason Baron * send data in SYN_RECV state. 389843f4a55SYuchung Cheng */ 39089278c9dSYuchung Cheng fastopen: 39111199369STonghao Zhang child = tcp_fastopen_create_child(sk, skb, req); 3927c85af88SEric Dumazet if (child) { 3939092a76dSJason Baron if (ret == 2) { 3949092a76dSJason Baron valid_foc.exp = foc->exp; 3959092a76dSJason Baron *foc = valid_foc; 3969092a76dSJason Baron NET_INC_STATS(sock_net(sk), 3979092a76dSJason Baron LINUX_MIB_TCPFASTOPENPASSIVEALTKEY); 3989092a76dSJason Baron } else { 39989278c9dSYuchung Cheng foc->len = -1; 4009092a76dSJason Baron } 401c10d9310SEric Dumazet NET_INC_STATS(sock_net(sk), 402843f4a55SYuchung Cheng LINUX_MIB_TCPFASTOPENPASSIVE); 4037c85af88SEric Dumazet return child; 4045b7ed089SYuchung Cheng } 4059092a76dSJason Baron NET_INC_STATS(sock_net(sk), 4069092a76dSJason Baron LINUX_MIB_TCPFASTOPENPASSIVEFAIL); 4079092a76dSJason Baron } 4089092a76dSJason Baron } 4097f9b838bSDaniel Lee valid_foc.exp = foc->exp; 41089278c9dSYuchung Cheng *foc = valid_foc; 4117c85af88SEric Dumazet return NULL; 4125b7ed089SYuchung Cheng } 413065263f4SWei Wang 414065263f4SWei Wang bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss, 415065263f4SWei Wang struct tcp_fastopen_cookie *cookie) 416065263f4SWei Wang { 41771c02379SChristoph Paasch const struct dst_entry *dst; 418065263f4SWei Wang 4197268586bSYuchung Cheng tcp_fastopen_cache_get(sk, mss, cookie); 420cf1ef3f0SWei Wang 421cf1ef3f0SWei Wang /* Firewall blackhole issue check */ 422cf1ef3f0SWei Wang if (tcp_fastopen_active_should_disable(sk)) { 423cf1ef3f0SWei Wang cookie->len = -1; 424cf1ef3f0SWei Wang return false; 425cf1ef3f0SWei Wang } 426cf1ef3f0SWei Wang 42771c02379SChristoph Paasch dst = __sk_dst_get(sk); 42871c02379SChristoph Paasch 42971c02379SChristoph Paasch if (tcp_fastopen_no_cookie(sk, dst, TFO_CLIENT_NO_COOKIE)) { 430065263f4SWei Wang cookie->len = -1; 431065263f4SWei Wang return true; 432065263f4SWei Wang } 43348027478SJason Baron if (cookie->len > 0) 43448027478SJason Baron return true; 43548027478SJason Baron tcp_sk(sk)->fastopen_client_fail = TFO_COOKIE_UNAVAILABLE; 43648027478SJason Baron return false; 437065263f4SWei Wang } 43819f6d3f3SWei Wang 43919f6d3f3SWei Wang /* This function checks if we want to defer sending SYN until the first 44019f6d3f3SWei Wang * write(). We defer under the following conditions: 44119f6d3f3SWei Wang * 1. fastopen_connect sockopt is set 44219f6d3f3SWei Wang * 2. we have a valid cookie 44319f6d3f3SWei Wang * Return value: return true if we want to defer until application writes data 44419f6d3f3SWei Wang * return false if we want to send out SYN immediately 44519f6d3f3SWei Wang */ 44619f6d3f3SWei Wang bool tcp_fastopen_defer_connect(struct sock *sk, int *err) 44719f6d3f3SWei Wang { 44819f6d3f3SWei Wang struct tcp_fastopen_cookie cookie = { .len = 0 }; 44919f6d3f3SWei Wang struct tcp_sock *tp = tcp_sk(sk); 45019f6d3f3SWei Wang u16 mss; 45119f6d3f3SWei Wang 45219f6d3f3SWei Wang if (tp->fastopen_connect && !tp->fastopen_req) { 45319f6d3f3SWei Wang if (tcp_fastopen_cookie_check(sk, &mss, &cookie)) { 45419f6d3f3SWei Wang inet_sk(sk)->defer_connect = 1; 45519f6d3f3SWei Wang return true; 45619f6d3f3SWei Wang } 45719f6d3f3SWei Wang 45819f6d3f3SWei Wang /* Alloc fastopen_req in order for FO option to be included 45919f6d3f3SWei Wang * in SYN 46019f6d3f3SWei Wang */ 46119f6d3f3SWei Wang tp->fastopen_req = kzalloc(sizeof(*tp->fastopen_req), 46219f6d3f3SWei Wang sk->sk_allocation); 46319f6d3f3SWei Wang if (tp->fastopen_req) 46419f6d3f3SWei Wang tp->fastopen_req->cookie = cookie; 46519f6d3f3SWei Wang else 46619f6d3f3SWei Wang *err = -ENOBUFS; 46719f6d3f3SWei Wang } 46819f6d3f3SWei Wang return false; 46919f6d3f3SWei Wang } 47019f6d3f3SWei Wang EXPORT_SYMBOL(tcp_fastopen_defer_connect); 471cf1ef3f0SWei Wang 472cf1ef3f0SWei Wang /* 473cf1ef3f0SWei Wang * The following code block is to deal with middle box issues with TFO: 474cf1ef3f0SWei Wang * Middlebox firewall issues can potentially cause server's data being 475cf1ef3f0SWei Wang * blackholed after a successful 3WHS using TFO. 476cf1ef3f0SWei Wang * The proposed solution is to disable active TFO globally under the 477cf1ef3f0SWei Wang * following circumstances: 478cf1ef3f0SWei Wang * 1. client side TFO socket receives out of order FIN 479cf1ef3f0SWei Wang * 2. client side TFO socket receives out of order RST 4807268586bSYuchung Cheng * 3. client side TFO socket has timed out three times consecutively during 4817268586bSYuchung Cheng * or after handshake 482cf1ef3f0SWei Wang * We disable active side TFO globally for 1hr at first. Then if it 483cf1ef3f0SWei Wang * happens again, we disable it for 2h, then 4h, 8h, ... 484cf1ef3f0SWei Wang * And we reset the timeout back to 1hr when we see a successful active 485cf1ef3f0SWei Wang * TFO connection with data exchanges. 486cf1ef3f0SWei Wang */ 487cf1ef3f0SWei Wang 488cf1ef3f0SWei Wang /* Disable active TFO and record current jiffies and 489cf1ef3f0SWei Wang * tfo_active_disable_times 490cf1ef3f0SWei Wang */ 49146c2fa39SWei Wang void tcp_fastopen_active_disable(struct sock *sk) 492cf1ef3f0SWei Wang { 4933733be14SHaishuang Yan struct net *net = sock_net(sk); 494cf1ef3f0SWei Wang 495021266ecSKuniyuki Iwashima if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen_blackhole_timeout)) 496213ad73dSWei Wang return; 497213ad73dSWei Wang 4986f20c8adSEric Dumazet /* Paired with READ_ONCE() in tcp_fastopen_active_should_disable() */ 4996f20c8adSEric Dumazet WRITE_ONCE(net->ipv4.tfo_active_disable_stamp, jiffies); 5006f20c8adSEric Dumazet 5016f20c8adSEric Dumazet /* Paired with smp_rmb() in tcp_fastopen_active_should_disable(). 5026f20c8adSEric Dumazet * We want net->ipv4.tfo_active_disable_stamp to be updated first. 5036f20c8adSEric Dumazet */ 5046f20c8adSEric Dumazet smp_mb__before_atomic(); 5053733be14SHaishuang Yan atomic_inc(&net->ipv4.tfo_active_disable_times); 5066f20c8adSEric Dumazet 5073733be14SHaishuang Yan NET_INC_STATS(net, LINUX_MIB_TCPFASTOPENBLACKHOLE); 508cf1ef3f0SWei Wang } 509cf1ef3f0SWei Wang 510cf1ef3f0SWei Wang /* Calculate timeout for tfo active disable 511cf1ef3f0SWei Wang * Return true if we are still in the active TFO disable period 512cf1ef3f0SWei Wang * Return false if timeout already expired and we should use active TFO 513cf1ef3f0SWei Wang */ 514cf1ef3f0SWei Wang bool tcp_fastopen_active_should_disable(struct sock *sk) 515cf1ef3f0SWei Wang { 516021266ecSKuniyuki Iwashima unsigned int tfo_bh_timeout = 517021266ecSKuniyuki Iwashima READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen_blackhole_timeout); 518cf1ef3f0SWei Wang unsigned long timeout; 519213ad73dSWei Wang int tfo_da_times; 5203733be14SHaishuang Yan int multiplier; 521cf1ef3f0SWei Wang 522213ad73dSWei Wang if (!tfo_bh_timeout) 523213ad73dSWei Wang return false; 524213ad73dSWei Wang 525213ad73dSWei Wang tfo_da_times = atomic_read(&sock_net(sk)->ipv4.tfo_active_disable_times); 526cf1ef3f0SWei Wang if (!tfo_da_times) 527cf1ef3f0SWei Wang return false; 528cf1ef3f0SWei Wang 5296f20c8adSEric Dumazet /* Paired with smp_mb__before_atomic() in tcp_fastopen_active_disable() */ 5306f20c8adSEric Dumazet smp_rmb(); 5316f20c8adSEric Dumazet 532974d8f86SZheng Yongjun /* Limit timeout to max: 2^6 * initial timeout */ 533cf1ef3f0SWei Wang multiplier = 1 << min(tfo_da_times - 1, 6); 5346f20c8adSEric Dumazet 5356f20c8adSEric Dumazet /* Paired with the WRITE_ONCE() in tcp_fastopen_active_disable(). */ 5366f20c8adSEric Dumazet timeout = READ_ONCE(sock_net(sk)->ipv4.tfo_active_disable_stamp) + 5376f20c8adSEric Dumazet multiplier * tfo_bh_timeout * HZ; 5386f20c8adSEric Dumazet if (time_before(jiffies, timeout)) 539cf1ef3f0SWei Wang return true; 540cf1ef3f0SWei Wang 541cf1ef3f0SWei Wang /* Mark check bit so we can check for successful active TFO 542cf1ef3f0SWei Wang * condition and reset tfo_active_disable_times 543cf1ef3f0SWei Wang */ 544cf1ef3f0SWei Wang tcp_sk(sk)->syn_fastopen_ch = 1; 545cf1ef3f0SWei Wang return false; 546cf1ef3f0SWei Wang } 547cf1ef3f0SWei Wang 548cf1ef3f0SWei Wang /* Disable active TFO if FIN is the only packet in the ofo queue 549cf1ef3f0SWei Wang * and no data is received. 550cf1ef3f0SWei Wang * Also check if we can reset tfo_active_disable_times if data is 551cf1ef3f0SWei Wang * received successfully on a marked active TFO sockets opened on 552cf1ef3f0SWei Wang * a non-loopback interface 553cf1ef3f0SWei Wang */ 554cf1ef3f0SWei Wang void tcp_fastopen_active_disable_ofo_check(struct sock *sk) 555cf1ef3f0SWei Wang { 556cf1ef3f0SWei Wang struct tcp_sock *tp = tcp_sk(sk); 557cf1ef3f0SWei Wang struct dst_entry *dst; 55818a4c0eaSEric Dumazet struct sk_buff *skb; 559cf1ef3f0SWei Wang 560cf1ef3f0SWei Wang if (!tp->syn_fastopen) 561cf1ef3f0SWei Wang return; 562cf1ef3f0SWei Wang 563cf1ef3f0SWei Wang if (!tp->data_segs_in) { 56418a4c0eaSEric Dumazet skb = skb_rb_first(&tp->out_of_order_queue); 56518a4c0eaSEric Dumazet if (skb && !skb_rb_next(skb)) { 566cf1ef3f0SWei Wang if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) { 56746c2fa39SWei Wang tcp_fastopen_active_disable(sk); 568cf1ef3f0SWei Wang return; 569cf1ef3f0SWei Wang } 570cf1ef3f0SWei Wang } 571cf1ef3f0SWei Wang } else if (tp->syn_fastopen_ch && 5723733be14SHaishuang Yan atomic_read(&sock_net(sk)->ipv4.tfo_active_disable_times)) { 573cf1ef3f0SWei Wang dst = sk_dst_get(sk); 574cf1ef3f0SWei Wang if (!(dst && dst->dev && (dst->dev->flags & IFF_LOOPBACK))) 5753733be14SHaishuang Yan atomic_set(&sock_net(sk)->ipv4.tfo_active_disable_times, 0); 576cf1ef3f0SWei Wang dst_release(dst); 577cf1ef3f0SWei Wang } 578cf1ef3f0SWei Wang } 5797268586bSYuchung Cheng 5807268586bSYuchung Cheng void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired) 5817268586bSYuchung Cheng { 5827268586bSYuchung Cheng u32 timeouts = inet_csk(sk)->icsk_retransmits; 5837268586bSYuchung Cheng struct tcp_sock *tp = tcp_sk(sk); 5847268586bSYuchung Cheng 5857268586bSYuchung Cheng /* Broken middle-boxes may black-hole Fast Open connection during or 5867268586bSYuchung Cheng * even after the handshake. Be extremely conservative and pause 5877268586bSYuchung Cheng * Fast Open globally after hitting the third consecutive timeout or 5887268586bSYuchung Cheng * exceeding the configured timeout limit. 5897268586bSYuchung Cheng */ 5907268586bSYuchung Cheng if ((tp->syn_fastopen || tp->syn_data || tp->syn_data_acked) && 5917268586bSYuchung Cheng (timeouts == 2 || (timeouts < 2 && expired))) { 5927268586bSYuchung Cheng tcp_fastopen_active_disable(sk); 5937268586bSYuchung Cheng NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFASTOPENACTIVEFAIL); 5947268586bSYuchung Cheng } 5957268586bSYuchung Cheng } 596