1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2018 Chelsio Communications, Inc. 4 * 5 * Written by: Atul Gupta (atul.gupta@chelsio.com) 6 */ 7 8 #include <linux/module.h> 9 #include <linux/list.h> 10 #include <linux/workqueue.h> 11 #include <linux/skbuff.h> 12 #include <linux/timer.h> 13 #include <linux/notifier.h> 14 #include <linux/inetdevice.h> 15 #include <linux/ip.h> 16 #include <linux/tcp.h> 17 #include <linux/sched/signal.h> 18 #include <linux/kallsyms.h> 19 #include <linux/kprobes.h> 20 #include <linux/if_vlan.h> 21 #include <linux/ipv6.h> 22 #include <net/ipv6.h> 23 #include <net/transp_v6.h> 24 #include <net/ip6_route.h> 25 #include <net/inet_common.h> 26 #include <net/tcp.h> 27 #include <net/dst.h> 28 #include <net/tls.h> 29 #include <net/addrconf.h> 30 #include <net/secure_seq.h> 31 32 #include "chtls.h" 33 #include "chtls_cm.h" 34 #include "clip_tbl.h" 35 #include "t4_tcb.h" 36 37 /* 38 * State transitions and actions for close. Note that if we are in SYN_SENT 39 * we remain in that state as we cannot control a connection while it's in 40 * SYN_SENT; such connections are allowed to establish and are then aborted. 41 */ 42 static unsigned char new_state[16] = { 43 /* current state: new state: action: */ 44 /* (Invalid) */ TCP_CLOSE, 45 /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN, 46 /* TCP_SYN_SENT */ TCP_SYN_SENT, 47 /* TCP_SYN_RECV */ TCP_FIN_WAIT1 | TCP_ACTION_FIN, 48 /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1, 49 /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2, 50 /* TCP_TIME_WAIT */ TCP_CLOSE, 51 /* TCP_CLOSE */ TCP_CLOSE, 52 /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN, 53 /* TCP_LAST_ACK */ TCP_LAST_ACK, 54 /* TCP_LISTEN */ TCP_CLOSE, 55 /* TCP_CLOSING */ TCP_CLOSING, 56 }; 57 58 static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev) 59 { 60 struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_ATOMIC); 61 62 if (!csk) 63 return NULL; 64 65 csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC); 66 if (!csk->txdata_skb_cache) { 67 kfree(csk); 68 return NULL; 69 } 70 71 kref_init(&csk->kref); 72 csk->cdev = cdev; 73 skb_queue_head_init(&csk->txq); 74 csk->wr_skb_head = NULL; 75 csk->wr_skb_tail = NULL; 76 csk->mss = MAX_MSS; 77 csk->tlshws.ofld = 1; 78 csk->tlshws.txkey = -1; 79 csk->tlshws.rxkey = -1; 80 csk->tlshws.mfs = TLS_MFS; 81 skb_queue_head_init(&csk->tlshws.sk_recv_queue); 82 return csk; 83 } 84 85 static void chtls_sock_release(struct kref *ref) 86 { 87 struct chtls_sock *csk = 88 container_of(ref, struct chtls_sock, kref); 89 90 kfree(csk); 91 } 92 93 static struct net_device *chtls_find_netdev(struct chtls_dev *cdev, 94 struct sock *sk) 95 { 96 struct adapter *adap = pci_get_drvdata(cdev->pdev); 97 struct net_device *ndev = cdev->ports[0]; 98 #if IS_ENABLED(CONFIG_IPV6) 99 struct net_device *temp; 100 int addr_type; 101 #endif 102 int i; 103 104 switch (sk->sk_family) { 105 case PF_INET: 106 if (likely(!inet_sk(sk)->inet_rcv_saddr)) 107 return ndev; 108 ndev = __ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr, false); 109 break; 110 #if IS_ENABLED(CONFIG_IPV6) 111 case PF_INET6: 112 addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr); 113 if (likely(addr_type == IPV6_ADDR_ANY)) 114 return ndev; 115 116 for_each_netdev_rcu(&init_net, temp) { 117 if (ipv6_chk_addr(&init_net, (struct in6_addr *) 118 &sk->sk_v6_rcv_saddr, temp, 1)) { 119 ndev = temp; 120 break; 121 } 122 } 123 break; 124 #endif 125 default: 126 return NULL; 127 } 128 129 if (!ndev) 130 return NULL; 131 132 if (is_vlan_dev(ndev)) 133 ndev = vlan_dev_real_dev(ndev); 134 135 for_each_port(adap, i) 136 if (cdev->ports[i] == ndev) 137 return ndev; 138 return NULL; 139 } 140 141 static void assign_rxopt(struct sock *sk, unsigned int opt) 142 { 143 const struct chtls_dev *cdev; 144 struct chtls_sock *csk; 145 struct tcp_sock *tp; 146 147 csk = rcu_dereference_sk_user_data(sk); 148 tp = tcp_sk(sk); 149 150 cdev = csk->cdev; 151 tp->tcp_header_len = sizeof(struct tcphdr); 152 tp->rx_opt.mss_clamp = cdev->mtus[TCPOPT_MSS_G(opt)] - 40; 153 tp->mss_cache = tp->rx_opt.mss_clamp; 154 tp->rx_opt.tstamp_ok = TCPOPT_TSTAMP_G(opt); 155 tp->rx_opt.snd_wscale = TCPOPT_SACK_G(opt); 156 tp->rx_opt.wscale_ok = TCPOPT_WSCALE_OK_G(opt); 157 SND_WSCALE(tp) = TCPOPT_SND_WSCALE_G(opt); 158 if (!tp->rx_opt.wscale_ok) 159 tp->rx_opt.rcv_wscale = 0; 160 if (tp->rx_opt.tstamp_ok) { 161 tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED; 162 tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED; 163 } else if (csk->opt2 & TSTAMPS_EN_F) { 164 csk->opt2 &= ~TSTAMPS_EN_F; 165 csk->mtu_idx = TCPOPT_MSS_G(opt); 166 } 167 } 168 169 static void chtls_purge_receive_queue(struct sock *sk) 170 { 171 struct sk_buff *skb; 172 173 while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) { 174 skb_dst_set(skb, (void *)NULL); 175 kfree_skb(skb); 176 } 177 } 178 179 static void chtls_purge_write_queue(struct sock *sk) 180 { 181 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); 182 struct sk_buff *skb; 183 184 while ((skb = __skb_dequeue(&csk->txq))) { 185 sk->sk_wmem_queued -= skb->truesize; 186 __kfree_skb(skb); 187 } 188 } 189 190 static void chtls_purge_recv_queue(struct sock *sk) 191 { 192 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); 193 struct chtls_hws *tlsk = &csk->tlshws; 194 struct sk_buff *skb; 195 196 while ((skb = __skb_dequeue(&tlsk->sk_recv_queue)) != NULL) { 197 skb_dst_set(skb, NULL); 198 kfree_skb(skb); 199 } 200 } 201 202 static void abort_arp_failure(void *handle, struct sk_buff *skb) 203 { 204 struct cpl_abort_req *req = cplhdr(skb); 205 struct chtls_dev *cdev; 206 207 cdev = (struct chtls_dev *)handle; 208 req->cmd = CPL_ABORT_NO_RST; 209 cxgb4_ofld_send(cdev->lldi->ports[0], skb); 210 } 211 212 static struct sk_buff *alloc_ctrl_skb(struct sk_buff *skb, int len) 213 { 214 if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) { 215 __skb_trim(skb, 0); 216 refcount_inc(&skb->users); 217 } else { 218 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL); 219 } 220 return skb; 221 } 222 223 static void chtls_send_abort(struct sock *sk, int mode, struct sk_buff *skb) 224 { 225 struct cpl_abort_req *req; 226 struct chtls_sock *csk; 227 struct tcp_sock *tp; 228 229 csk = rcu_dereference_sk_user_data(sk); 230 tp = tcp_sk(sk); 231 232 if (!skb) 233 skb = alloc_ctrl_skb(csk->txdata_skb_cache, sizeof(*req)); 234 235 req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req)); 236 INIT_TP_WR_CPL(req, CPL_ABORT_REQ, csk->tid); 237 skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA); 238 req->rsvd0 = htonl(tp->snd_nxt); 239 req->rsvd1 = !csk_flag_nochk(csk, CSK_TX_DATA_SENT); 240 req->cmd = mode; 241 t4_set_arp_err_handler(skb, csk->cdev, abort_arp_failure); 242 send_or_defer(sk, tp, skb, mode == CPL_ABORT_SEND_RST); 243 } 244 245 static void chtls_send_reset(struct sock *sk, int mode, struct sk_buff *skb) 246 { 247 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); 248 249 if (unlikely(csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) || 250 !csk->cdev)) { 251 if (sk->sk_state == TCP_SYN_RECV) 252 csk_set_flag(csk, CSK_RST_ABORTED); 253 goto out; 254 } 255 256 if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) { 257 struct tcp_sock *tp = tcp_sk(sk); 258 259 if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0) 260 WARN_ONCE(1, "send tx flowc error"); 261 csk_set_flag(csk, CSK_TX_DATA_SENT); 262 } 263 264 csk_set_flag(csk, CSK_ABORT_RPL_PENDING); 265 chtls_purge_write_queue(sk); 266 267 csk_set_flag(csk, CSK_ABORT_SHUTDOWN); 268 if (sk->sk_state != TCP_SYN_RECV) 269 chtls_send_abort(sk, mode, skb); 270 else 271 chtls_set_tcb_field_rpl_skb(sk, TCB_T_FLAGS_W, 272 TCB_T_FLAGS_V(TCB_T_FLAGS_M), 0, 273 TCB_FIELD_COOKIE_TFLAG, 1); 274 275 return; 276 out: 277 kfree_skb(skb); 278 } 279 280 static void release_tcp_port(struct sock *sk) 281 { 282 if (inet_csk(sk)->icsk_bind_hash) 283 inet_put_port(sk); 284 } 285 286 static void tcp_uncork(struct sock *sk) 287 { 288 struct tcp_sock *tp = tcp_sk(sk); 289 290 if (tp->nonagle & TCP_NAGLE_CORK) { 291 tp->nonagle &= ~TCP_NAGLE_CORK; 292 chtls_tcp_push(sk, 0); 293 } 294 } 295 296 static void chtls_close_conn(struct sock *sk) 297 { 298 struct cpl_close_con_req *req; 299 struct chtls_sock *csk; 300 struct sk_buff *skb; 301 unsigned int tid; 302 unsigned int len; 303 304 len = roundup(sizeof(struct cpl_close_con_req), 16); 305 csk = rcu_dereference_sk_user_data(sk); 306 tid = csk->tid; 307 308 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL); 309 req = (struct cpl_close_con_req *)__skb_put(skb, len); 310 memset(req, 0, len); 311 req->wr.wr_hi = htonl(FW_WR_OP_V(FW_TP_WR) | 312 FW_WR_IMMDLEN_V(sizeof(*req) - 313 sizeof(req->wr))); 314 req->wr.wr_mid = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)) | 315 FW_WR_FLOWID_V(tid)); 316 317 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); 318 319 tcp_uncork(sk); 320 skb_entail(sk, skb, ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND); 321 if (sk->sk_state != TCP_SYN_SENT) 322 chtls_push_frames(csk, 1); 323 } 324 325 /* 326 * Perform a state transition during close and return the actions indicated 327 * for the transition. Do not make this function inline, the main reason 328 * it exists at all is to avoid multiple inlining of tcp_set_state. 329 */ 330 static int make_close_transition(struct sock *sk) 331 { 332 int next = (int)new_state[sk->sk_state]; 333 334 tcp_set_state(sk, next & TCP_STATE_MASK); 335 return next & TCP_ACTION_FIN; 336 } 337 338 void chtls_close(struct sock *sk, long timeout) 339 { 340 int data_lost, prev_state; 341 struct chtls_sock *csk; 342 343 csk = rcu_dereference_sk_user_data(sk); 344 345 lock_sock(sk); 346 sk->sk_shutdown |= SHUTDOWN_MASK; 347 348 data_lost = skb_queue_len(&sk->sk_receive_queue); 349 data_lost |= skb_queue_len(&csk->tlshws.sk_recv_queue); 350 chtls_purge_recv_queue(sk); 351 chtls_purge_receive_queue(sk); 352 353 if (sk->sk_state == TCP_CLOSE) { 354 goto wait; 355 } else if (data_lost || sk->sk_state == TCP_SYN_SENT) { 356 chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL); 357 release_tcp_port(sk); 358 goto unlock; 359 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) { 360 sk->sk_prot->disconnect(sk, 0); 361 } else if (make_close_transition(sk)) { 362 chtls_close_conn(sk); 363 } 364 wait: 365 if (timeout) 366 sk_stream_wait_close(sk, timeout); 367 368 unlock: 369 prev_state = sk->sk_state; 370 sock_hold(sk); 371 sock_orphan(sk); 372 373 release_sock(sk); 374 375 local_bh_disable(); 376 bh_lock_sock(sk); 377 378 if (prev_state != TCP_CLOSE && sk->sk_state == TCP_CLOSE) 379 goto out; 380 381 if (sk->sk_state == TCP_FIN_WAIT2 && tcp_sk(sk)->linger2 < 0 && 382 !csk_flag(sk, CSK_ABORT_SHUTDOWN)) { 383 struct sk_buff *skb; 384 385 skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC); 386 if (skb) 387 chtls_send_reset(sk, CPL_ABORT_SEND_RST, skb); 388 } 389 390 if (sk->sk_state == TCP_CLOSE) 391 inet_csk_destroy_sock(sk); 392 393 out: 394 bh_unlock_sock(sk); 395 local_bh_enable(); 396 sock_put(sk); 397 } 398 399 /* 400 * Wait until a socket enters on of the given states. 401 */ 402 static int wait_for_states(struct sock *sk, unsigned int states) 403 { 404 DECLARE_WAITQUEUE(wait, current); 405 struct socket_wq _sk_wq; 406 long current_timeo; 407 int err = 0; 408 409 current_timeo = 200; 410 411 /* 412 * We want this to work even when there's no associated struct socket. 413 * In that case we provide a temporary wait_queue_head_t. 414 */ 415 if (!sk->sk_wq) { 416 init_waitqueue_head(&_sk_wq.wait); 417 _sk_wq.fasync_list = NULL; 418 init_rcu_head_on_stack(&_sk_wq.rcu); 419 RCU_INIT_POINTER(sk->sk_wq, &_sk_wq); 420 } 421 422 add_wait_queue(sk_sleep(sk), &wait); 423 while (!sk_in_state(sk, states)) { 424 if (!current_timeo) { 425 err = -EBUSY; 426 break; 427 } 428 if (signal_pending(current)) { 429 err = sock_intr_errno(current_timeo); 430 break; 431 } 432 set_current_state(TASK_UNINTERRUPTIBLE); 433 release_sock(sk); 434 if (!sk_in_state(sk, states)) 435 current_timeo = schedule_timeout(current_timeo); 436 __set_current_state(TASK_RUNNING); 437 lock_sock(sk); 438 } 439 remove_wait_queue(sk_sleep(sk), &wait); 440 441 if (rcu_dereference(sk->sk_wq) == &_sk_wq) 442 sk->sk_wq = NULL; 443 return err; 444 } 445 446 int chtls_disconnect(struct sock *sk, int flags) 447 { 448 struct tcp_sock *tp; 449 int err; 450 451 tp = tcp_sk(sk); 452 chtls_purge_recv_queue(sk); 453 chtls_purge_receive_queue(sk); 454 chtls_purge_write_queue(sk); 455 456 if (sk->sk_state != TCP_CLOSE) { 457 sk->sk_err = ECONNRESET; 458 chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL); 459 err = wait_for_states(sk, TCPF_CLOSE); 460 if (err) 461 return err; 462 } 463 chtls_purge_recv_queue(sk); 464 chtls_purge_receive_queue(sk); 465 tp->max_window = 0xFFFF << (tp->rx_opt.snd_wscale); 466 return tcp_disconnect(sk, flags); 467 } 468 469 #define SHUTDOWN_ELIGIBLE_STATE (TCPF_ESTABLISHED | \ 470 TCPF_SYN_RECV | TCPF_CLOSE_WAIT) 471 void chtls_shutdown(struct sock *sk, int how) 472 { 473 if ((how & SEND_SHUTDOWN) && 474 sk_in_state(sk, SHUTDOWN_ELIGIBLE_STATE) && 475 make_close_transition(sk)) 476 chtls_close_conn(sk); 477 } 478 479 void chtls_destroy_sock(struct sock *sk) 480 { 481 struct chtls_sock *csk; 482 483 csk = rcu_dereference_sk_user_data(sk); 484 chtls_purge_recv_queue(sk); 485 csk->ulp_mode = ULP_MODE_NONE; 486 chtls_purge_write_queue(sk); 487 free_tls_keyid(sk); 488 kref_put(&csk->kref, chtls_sock_release); 489 if (sk->sk_family == AF_INET) 490 sk->sk_prot = &tcp_prot; 491 #if IS_ENABLED(CONFIG_IPV6) 492 else 493 sk->sk_prot = &tcpv6_prot; 494 #endif 495 sk->sk_prot->destroy(sk); 496 } 497 498 static void reset_listen_child(struct sock *child) 499 { 500 struct chtls_sock *csk = rcu_dereference_sk_user_data(child); 501 struct sk_buff *skb; 502 503 skb = alloc_ctrl_skb(csk->txdata_skb_cache, 504 sizeof(struct cpl_abort_req)); 505 506 chtls_send_reset(child, CPL_ABORT_SEND_RST, skb); 507 sock_orphan(child); 508 INC_ORPHAN_COUNT(child); 509 if (child->sk_state == TCP_CLOSE) 510 inet_csk_destroy_sock(child); 511 } 512 513 static void chtls_disconnect_acceptq(struct sock *listen_sk) 514 { 515 struct request_sock **pprev; 516 517 pprev = ACCEPT_QUEUE(listen_sk); 518 while (*pprev) { 519 struct request_sock *req = *pprev; 520 521 if (req->rsk_ops == &chtls_rsk_ops || 522 req->rsk_ops == &chtls_rsk_opsv6) { 523 struct sock *child = req->sk; 524 525 *pprev = req->dl_next; 526 sk_acceptq_removed(listen_sk); 527 reqsk_put(req); 528 sock_hold(child); 529 local_bh_disable(); 530 bh_lock_sock(child); 531 release_tcp_port(child); 532 reset_listen_child(child); 533 bh_unlock_sock(child); 534 local_bh_enable(); 535 sock_put(child); 536 } else { 537 pprev = &req->dl_next; 538 } 539 } 540 } 541 542 static int listen_hashfn(const struct sock *sk) 543 { 544 return ((unsigned long)sk >> 10) & (LISTEN_INFO_HASH_SIZE - 1); 545 } 546 547 static struct listen_info *listen_hash_add(struct chtls_dev *cdev, 548 struct sock *sk, 549 unsigned int stid) 550 { 551 struct listen_info *p = kmalloc(sizeof(*p), GFP_KERNEL); 552 553 if (p) { 554 int key = listen_hashfn(sk); 555 556 p->sk = sk; 557 p->stid = stid; 558 spin_lock(&cdev->listen_lock); 559 p->next = cdev->listen_hash_tab[key]; 560 cdev->listen_hash_tab[key] = p; 561 spin_unlock(&cdev->listen_lock); 562 } 563 return p; 564 } 565 566 static int listen_hash_find(struct chtls_dev *cdev, 567 struct sock *sk) 568 { 569 struct listen_info *p; 570 int stid = -1; 571 int key; 572 573 key = listen_hashfn(sk); 574 575 spin_lock(&cdev->listen_lock); 576 for (p = cdev->listen_hash_tab[key]; p; p = p->next) 577 if (p->sk == sk) { 578 stid = p->stid; 579 break; 580 } 581 spin_unlock(&cdev->listen_lock); 582 return stid; 583 } 584 585 static int listen_hash_del(struct chtls_dev *cdev, 586 struct sock *sk) 587 { 588 struct listen_info *p, **prev; 589 int stid = -1; 590 int key; 591 592 key = listen_hashfn(sk); 593 prev = &cdev->listen_hash_tab[key]; 594 595 spin_lock(&cdev->listen_lock); 596 for (p = *prev; p; prev = &p->next, p = p->next) 597 if (p->sk == sk) { 598 stid = p->stid; 599 *prev = p->next; 600 kfree(p); 601 break; 602 } 603 spin_unlock(&cdev->listen_lock); 604 return stid; 605 } 606 607 static void cleanup_syn_rcv_conn(struct sock *child, struct sock *parent) 608 { 609 struct request_sock *req; 610 struct chtls_sock *csk; 611 612 csk = rcu_dereference_sk_user_data(child); 613 req = csk->passive_reap_next; 614 615 reqsk_queue_removed(&inet_csk(parent)->icsk_accept_queue, req); 616 __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq); 617 chtls_reqsk_free(req); 618 csk->passive_reap_next = NULL; 619 } 620 621 static void chtls_reset_synq(struct listen_ctx *listen_ctx) 622 { 623 struct sock *listen_sk = listen_ctx->lsk; 624 625 while (!skb_queue_empty(&listen_ctx->synq)) { 626 struct chtls_sock *csk = 627 container_of((struct synq *)skb_peek 628 (&listen_ctx->synq), struct chtls_sock, synq); 629 struct sock *child = csk->sk; 630 631 cleanup_syn_rcv_conn(child, listen_sk); 632 sock_hold(child); 633 local_bh_disable(); 634 bh_lock_sock(child); 635 release_tcp_port(child); 636 reset_listen_child(child); 637 bh_unlock_sock(child); 638 local_bh_enable(); 639 sock_put(child); 640 } 641 } 642 643 int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk) 644 { 645 struct net_device *ndev; 646 #if IS_ENABLED(CONFIG_IPV6) 647 bool clip_valid = false; 648 #endif 649 struct listen_ctx *ctx; 650 struct adapter *adap; 651 struct port_info *pi; 652 int ret = 0; 653 int stid; 654 655 rcu_read_lock(); 656 ndev = chtls_find_netdev(cdev, sk); 657 rcu_read_unlock(); 658 if (!ndev) 659 return -EBADF; 660 661 pi = netdev_priv(ndev); 662 adap = pi->adapter; 663 if (!(adap->flags & CXGB4_FULL_INIT_DONE)) 664 return -EBADF; 665 666 if (listen_hash_find(cdev, sk) >= 0) /* already have it */ 667 return -EADDRINUSE; 668 669 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); 670 if (!ctx) 671 return -ENOMEM; 672 673 __module_get(THIS_MODULE); 674 ctx->lsk = sk; 675 ctx->cdev = cdev; 676 ctx->state = T4_LISTEN_START_PENDING; 677 skb_queue_head_init(&ctx->synq); 678 679 stid = cxgb4_alloc_stid(cdev->tids, sk->sk_family, ctx); 680 if (stid < 0) 681 goto free_ctx; 682 683 sock_hold(sk); 684 if (!listen_hash_add(cdev, sk, stid)) 685 goto free_stid; 686 687 if (sk->sk_family == PF_INET) { 688 ret = cxgb4_create_server(ndev, stid, 689 inet_sk(sk)->inet_rcv_saddr, 690 inet_sk(sk)->inet_sport, 0, 691 cdev->lldi->rxq_ids[0]); 692 #if IS_ENABLED(CONFIG_IPV6) 693 } else { 694 int addr_type; 695 696 addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr); 697 if (addr_type != IPV6_ADDR_ANY) { 698 ret = cxgb4_clip_get(ndev, (const u32 *) 699 &sk->sk_v6_rcv_saddr, 1); 700 if (ret) 701 goto del_hash; 702 clip_valid = true; 703 } 704 ret = cxgb4_create_server6(ndev, stid, 705 &sk->sk_v6_rcv_saddr, 706 inet_sk(sk)->inet_sport, 707 cdev->lldi->rxq_ids[0]); 708 #endif 709 } 710 if (ret > 0) 711 ret = net_xmit_errno(ret); 712 if (ret) 713 goto del_hash; 714 return 0; 715 del_hash: 716 #if IS_ENABLED(CONFIG_IPV6) 717 if (clip_valid) 718 cxgb4_clip_release(ndev, (const u32 *)&sk->sk_v6_rcv_saddr, 1); 719 #endif 720 listen_hash_del(cdev, sk); 721 free_stid: 722 cxgb4_free_stid(cdev->tids, stid, sk->sk_family); 723 sock_put(sk); 724 free_ctx: 725 kfree(ctx); 726 module_put(THIS_MODULE); 727 return -EBADF; 728 } 729 730 void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk) 731 { 732 struct listen_ctx *listen_ctx; 733 int stid; 734 735 stid = listen_hash_del(cdev, sk); 736 if (stid < 0) 737 return; 738 739 listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid); 740 chtls_reset_synq(listen_ctx); 741 742 cxgb4_remove_server(cdev->lldi->ports[0], stid, 743 cdev->lldi->rxq_ids[0], sk->sk_family == PF_INET6); 744 745 #if IS_ENABLED(CONFIG_IPV6) 746 if (sk->sk_family == PF_INET6) { 747 struct net_device *ndev = chtls_find_netdev(cdev, sk); 748 int addr_type = 0; 749 750 addr_type = ipv6_addr_type((const struct in6_addr *) 751 &sk->sk_v6_rcv_saddr); 752 if (addr_type != IPV6_ADDR_ANY) 753 cxgb4_clip_release(ndev, (const u32 *) 754 &sk->sk_v6_rcv_saddr, 1); 755 } 756 #endif 757 chtls_disconnect_acceptq(sk); 758 } 759 760 static int chtls_pass_open_rpl(struct chtls_dev *cdev, struct sk_buff *skb) 761 { 762 struct cpl_pass_open_rpl *rpl = cplhdr(skb) + RSS_HDR; 763 unsigned int stid = GET_TID(rpl); 764 struct listen_ctx *listen_ctx; 765 766 listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid); 767 if (!listen_ctx) 768 return CPL_RET_BUF_DONE; 769 770 if (listen_ctx->state == T4_LISTEN_START_PENDING) { 771 listen_ctx->state = T4_LISTEN_STARTED; 772 return CPL_RET_BUF_DONE; 773 } 774 775 if (rpl->status != CPL_ERR_NONE) { 776 pr_info("Unexpected PASS_OPEN_RPL status %u for STID %u\n", 777 rpl->status, stid); 778 } else { 779 cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family); 780 sock_put(listen_ctx->lsk); 781 kfree(listen_ctx); 782 module_put(THIS_MODULE); 783 } 784 return CPL_RET_BUF_DONE; 785 } 786 787 static int chtls_close_listsrv_rpl(struct chtls_dev *cdev, struct sk_buff *skb) 788 { 789 struct cpl_close_listsvr_rpl *rpl = cplhdr(skb) + RSS_HDR; 790 struct listen_ctx *listen_ctx; 791 unsigned int stid; 792 void *data; 793 794 stid = GET_TID(rpl); 795 data = lookup_stid(cdev->tids, stid); 796 listen_ctx = (struct listen_ctx *)data; 797 798 if (rpl->status != CPL_ERR_NONE) { 799 pr_info("Unexpected CLOSE_LISTSRV_RPL status %u for STID %u\n", 800 rpl->status, stid); 801 } else { 802 cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family); 803 sock_put(listen_ctx->lsk); 804 kfree(listen_ctx); 805 module_put(THIS_MODULE); 806 } 807 return CPL_RET_BUF_DONE; 808 } 809 810 static void chtls_purge_wr_queue(struct sock *sk) 811 { 812 struct sk_buff *skb; 813 814 while ((skb = dequeue_wr(sk)) != NULL) 815 kfree_skb(skb); 816 } 817 818 static void chtls_release_resources(struct sock *sk) 819 { 820 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); 821 struct chtls_dev *cdev = csk->cdev; 822 unsigned int tid = csk->tid; 823 struct tid_info *tids; 824 825 if (!cdev) 826 return; 827 828 tids = cdev->tids; 829 kfree_skb(csk->txdata_skb_cache); 830 csk->txdata_skb_cache = NULL; 831 832 if (csk->wr_credits != csk->wr_max_credits) { 833 chtls_purge_wr_queue(sk); 834 chtls_reset_wr_list(csk); 835 } 836 837 if (csk->l2t_entry) { 838 cxgb4_l2t_release(csk->l2t_entry); 839 csk->l2t_entry = NULL; 840 } 841 842 if (sk->sk_state != TCP_SYN_SENT) { 843 cxgb4_remove_tid(tids, csk->port_id, tid, sk->sk_family); 844 sock_put(sk); 845 } 846 } 847 848 static void chtls_conn_done(struct sock *sk) 849 { 850 if (sock_flag(sk, SOCK_DEAD)) 851 chtls_purge_receive_queue(sk); 852 sk_wakeup_sleepers(sk, 0); 853 tcp_done(sk); 854 } 855 856 static void do_abort_syn_rcv(struct sock *child, struct sock *parent) 857 { 858 /* 859 * If the server is still open we clean up the child connection, 860 * otherwise the server already did the clean up as it was purging 861 * its SYN queue and the skb was just sitting in its backlog. 862 */ 863 if (likely(parent->sk_state == TCP_LISTEN)) { 864 cleanup_syn_rcv_conn(child, parent); 865 /* Without the below call to sock_orphan, 866 * we leak the socket resource with syn_flood test 867 * as inet_csk_destroy_sock will not be called 868 * in tcp_done since SOCK_DEAD flag is not set. 869 * Kernel handles this differently where new socket is 870 * created only after 3 way handshake is done. 871 */ 872 sock_orphan(child); 873 percpu_counter_inc((child)->sk_prot->orphan_count); 874 chtls_release_resources(child); 875 chtls_conn_done(child); 876 } else { 877 if (csk_flag(child, CSK_RST_ABORTED)) { 878 chtls_release_resources(child); 879 chtls_conn_done(child); 880 } 881 } 882 } 883 884 static void pass_open_abort(struct sock *child, struct sock *parent, 885 struct sk_buff *skb) 886 { 887 do_abort_syn_rcv(child, parent); 888 kfree_skb(skb); 889 } 890 891 static void bl_pass_open_abort(struct sock *lsk, struct sk_buff *skb) 892 { 893 pass_open_abort(skb->sk, lsk, skb); 894 } 895 896 static void chtls_pass_open_arp_failure(struct sock *sk, 897 struct sk_buff *skb) 898 { 899 const struct request_sock *oreq; 900 struct chtls_sock *csk; 901 struct chtls_dev *cdev; 902 struct sock *parent; 903 void *data; 904 905 csk = rcu_dereference_sk_user_data(sk); 906 cdev = csk->cdev; 907 908 /* 909 * If the connection is being aborted due to the parent listening 910 * socket going away there's nothing to do, the ABORT_REQ will close 911 * the connection. 912 */ 913 if (csk_flag(sk, CSK_ABORT_RPL_PENDING)) { 914 kfree_skb(skb); 915 return; 916 } 917 918 oreq = csk->passive_reap_next; 919 data = lookup_stid(cdev->tids, oreq->ts_recent); 920 parent = ((struct listen_ctx *)data)->lsk; 921 922 bh_lock_sock(parent); 923 if (!sock_owned_by_user(parent)) { 924 pass_open_abort(sk, parent, skb); 925 } else { 926 BLOG_SKB_CB(skb)->backlog_rcv = bl_pass_open_abort; 927 __sk_add_backlog(parent, skb); 928 } 929 bh_unlock_sock(parent); 930 } 931 932 static void chtls_accept_rpl_arp_failure(void *handle, 933 struct sk_buff *skb) 934 { 935 struct sock *sk = (struct sock *)handle; 936 937 sock_hold(sk); 938 process_cpl_msg(chtls_pass_open_arp_failure, sk, skb); 939 sock_put(sk); 940 } 941 942 static unsigned int chtls_select_mss(const struct chtls_sock *csk, 943 unsigned int pmtu, 944 struct cpl_pass_accept_req *req) 945 { 946 struct chtls_dev *cdev; 947 struct dst_entry *dst; 948 unsigned int tcpoptsz; 949 unsigned int iphdrsz; 950 unsigned int mtu_idx; 951 struct tcp_sock *tp; 952 unsigned int mss; 953 struct sock *sk; 954 955 mss = ntohs(req->tcpopt.mss); 956 sk = csk->sk; 957 dst = __sk_dst_get(sk); 958 cdev = csk->cdev; 959 tp = tcp_sk(sk); 960 tcpoptsz = 0; 961 962 #if IS_ENABLED(CONFIG_IPV6) 963 if (sk->sk_family == AF_INET6) 964 iphdrsz = sizeof(struct ipv6hdr) + sizeof(struct tcphdr); 965 else 966 #endif 967 iphdrsz = sizeof(struct iphdr) + sizeof(struct tcphdr); 968 if (req->tcpopt.tstamp) 969 tcpoptsz += round_up(TCPOLEN_TIMESTAMP, 4); 970 971 tp->advmss = dst_metric_advmss(dst); 972 if (USER_MSS(tp) && tp->advmss > USER_MSS(tp)) 973 tp->advmss = USER_MSS(tp); 974 if (tp->advmss > pmtu - iphdrsz) 975 tp->advmss = pmtu - iphdrsz; 976 if (mss && tp->advmss > mss) 977 tp->advmss = mss; 978 979 tp->advmss = cxgb4_best_aligned_mtu(cdev->lldi->mtus, 980 iphdrsz + tcpoptsz, 981 tp->advmss - tcpoptsz, 982 8, &mtu_idx); 983 tp->advmss -= iphdrsz; 984 985 inet_csk(sk)->icsk_pmtu_cookie = pmtu; 986 return mtu_idx; 987 } 988 989 static unsigned int select_rcv_wscale(int space, int wscale_ok, int win_clamp) 990 { 991 int wscale = 0; 992 993 if (space > MAX_RCV_WND) 994 space = MAX_RCV_WND; 995 if (win_clamp && win_clamp < space) 996 space = win_clamp; 997 998 if (wscale_ok) { 999 while (wscale < 14 && (65535 << wscale) < space) 1000 wscale++; 1001 } 1002 return wscale; 1003 } 1004 1005 static void chtls_pass_accept_rpl(struct sk_buff *skb, 1006 struct cpl_pass_accept_req *req, 1007 unsigned int tid) 1008 1009 { 1010 struct cpl_t5_pass_accept_rpl *rpl5; 1011 struct cxgb4_lld_info *lldi; 1012 const struct tcphdr *tcph; 1013 const struct tcp_sock *tp; 1014 struct chtls_sock *csk; 1015 unsigned int len; 1016 struct sock *sk; 1017 u32 opt2, hlen; 1018 u64 opt0; 1019 1020 sk = skb->sk; 1021 tp = tcp_sk(sk); 1022 csk = sk->sk_user_data; 1023 csk->tid = tid; 1024 lldi = csk->cdev->lldi; 1025 len = roundup(sizeof(*rpl5), 16); 1026 1027 rpl5 = __skb_put_zero(skb, len); 1028 INIT_TP_WR(rpl5, tid); 1029 1030 OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, 1031 csk->tid)); 1032 csk->mtu_idx = chtls_select_mss(csk, dst_mtu(__sk_dst_get(sk)), 1033 req); 1034 opt0 = TCAM_BYPASS_F | 1035 WND_SCALE_V(RCV_WSCALE(tp)) | 1036 MSS_IDX_V(csk->mtu_idx) | 1037 L2T_IDX_V(csk->l2t_entry->idx) | 1038 NAGLE_V(!(tp->nonagle & TCP_NAGLE_OFF)) | 1039 TX_CHAN_V(csk->tx_chan) | 1040 SMAC_SEL_V(csk->smac_idx) | 1041 DSCP_V(csk->tos >> 2) | 1042 ULP_MODE_V(ULP_MODE_TLS) | 1043 RCV_BUFSIZ_V(min(tp->rcv_wnd >> 10, RCV_BUFSIZ_M)); 1044 1045 opt2 = RX_CHANNEL_V(0) | 1046 RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid); 1047 1048 if (!is_t5(lldi->adapter_type)) 1049 opt2 |= RX_FC_DISABLE_F; 1050 if (req->tcpopt.tstamp) 1051 opt2 |= TSTAMPS_EN_F; 1052 if (req->tcpopt.sack) 1053 opt2 |= SACK_EN_F; 1054 hlen = ntohl(req->hdr_len); 1055 1056 tcph = (struct tcphdr *)((u8 *)(req + 1) + 1057 T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen)); 1058 if (tcph->ece && tcph->cwr) 1059 opt2 |= CCTRL_ECN_V(1); 1060 opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO); 1061 opt2 |= T5_ISS_F; 1062 opt2 |= T5_OPT_2_VALID_F; 1063 opt2 |= WND_SCALE_EN_V(WSCALE_OK(tp)); 1064 rpl5->opt0 = cpu_to_be64(opt0); 1065 rpl5->opt2 = cpu_to_be32(opt2); 1066 rpl5->iss = cpu_to_be32((prandom_u32() & ~7UL) - 1); 1067 set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id); 1068 t4_set_arp_err_handler(skb, sk, chtls_accept_rpl_arp_failure); 1069 cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry); 1070 } 1071 1072 static void inet_inherit_port(struct inet_hashinfo *hash_info, 1073 struct sock *lsk, struct sock *newsk) 1074 { 1075 local_bh_disable(); 1076 __inet_inherit_port(lsk, newsk); 1077 local_bh_enable(); 1078 } 1079 1080 static int chtls_backlog_rcv(struct sock *sk, struct sk_buff *skb) 1081 { 1082 if (skb->protocol) { 1083 kfree_skb(skb); 1084 return 0; 1085 } 1086 BLOG_SKB_CB(skb)->backlog_rcv(sk, skb); 1087 return 0; 1088 } 1089 1090 static void chtls_set_tcp_window(struct chtls_sock *csk) 1091 { 1092 struct net_device *ndev = csk->egress_dev; 1093 struct port_info *pi = netdev_priv(ndev); 1094 unsigned int linkspeed; 1095 u8 scale; 1096 1097 linkspeed = pi->link_cfg.speed; 1098 scale = linkspeed / SPEED_10000; 1099 #define CHTLS_10G_RCVWIN (256 * 1024) 1100 csk->rcv_win = CHTLS_10G_RCVWIN; 1101 if (scale) 1102 csk->rcv_win *= scale; 1103 #define CHTLS_10G_SNDWIN (256 * 1024) 1104 csk->snd_win = CHTLS_10G_SNDWIN; 1105 if (scale) 1106 csk->snd_win *= scale; 1107 } 1108 1109 static struct sock *chtls_recv_sock(struct sock *lsk, 1110 struct request_sock *oreq, 1111 void *network_hdr, 1112 const struct cpl_pass_accept_req *req, 1113 struct chtls_dev *cdev) 1114 { 1115 struct adapter *adap = pci_get_drvdata(cdev->pdev); 1116 struct neighbour *n = NULL; 1117 struct inet_sock *newinet; 1118 const struct iphdr *iph; 1119 struct tls_context *ctx; 1120 struct net_device *ndev; 1121 struct chtls_sock *csk; 1122 struct dst_entry *dst; 1123 struct tcp_sock *tp; 1124 struct sock *newsk; 1125 bool found = false; 1126 u16 port_id; 1127 int rxq_idx; 1128 int step, i; 1129 1130 iph = (const struct iphdr *)network_hdr; 1131 newsk = tcp_create_openreq_child(lsk, oreq, cdev->askb); 1132 if (!newsk) 1133 goto free_oreq; 1134 1135 if (lsk->sk_family == AF_INET) { 1136 dst = inet_csk_route_child_sock(lsk, newsk, oreq); 1137 if (!dst) 1138 goto free_sk; 1139 1140 n = dst_neigh_lookup(dst, &iph->saddr); 1141 #if IS_ENABLED(CONFIG_IPV6) 1142 } else { 1143 const struct ipv6hdr *ip6h; 1144 struct flowi6 fl6; 1145 1146 ip6h = (const struct ipv6hdr *)network_hdr; 1147 memset(&fl6, 0, sizeof(fl6)); 1148 fl6.flowi6_proto = IPPROTO_TCP; 1149 fl6.saddr = ip6h->daddr; 1150 fl6.daddr = ip6h->saddr; 1151 fl6.fl6_dport = inet_rsk(oreq)->ir_rmt_port; 1152 fl6.fl6_sport = htons(inet_rsk(oreq)->ir_num); 1153 security_req_classify_flow(oreq, flowi6_to_flowi_common(&fl6)); 1154 dst = ip6_dst_lookup_flow(sock_net(lsk), lsk, &fl6, NULL); 1155 if (IS_ERR(dst)) 1156 goto free_sk; 1157 n = dst_neigh_lookup(dst, &ip6h->saddr); 1158 #endif 1159 } 1160 if (!n || !n->dev) 1161 goto free_sk; 1162 1163 ndev = n->dev; 1164 if (!ndev) 1165 goto free_dst; 1166 if (is_vlan_dev(ndev)) 1167 ndev = vlan_dev_real_dev(ndev); 1168 1169 for_each_port(adap, i) 1170 if (cdev->ports[i] == ndev) 1171 found = true; 1172 1173 if (!found) 1174 goto free_dst; 1175 1176 port_id = cxgb4_port_idx(ndev); 1177 1178 csk = chtls_sock_create(cdev); 1179 if (!csk) 1180 goto free_dst; 1181 1182 csk->l2t_entry = cxgb4_l2t_get(cdev->lldi->l2t, n, ndev, 0); 1183 if (!csk->l2t_entry) 1184 goto free_csk; 1185 1186 newsk->sk_user_data = csk; 1187 newsk->sk_backlog_rcv = chtls_backlog_rcv; 1188 1189 tp = tcp_sk(newsk); 1190 newinet = inet_sk(newsk); 1191 1192 if (iph->version == 0x4) { 1193 newinet->inet_daddr = iph->saddr; 1194 newinet->inet_rcv_saddr = iph->daddr; 1195 newinet->inet_saddr = iph->daddr; 1196 #if IS_ENABLED(CONFIG_IPV6) 1197 } else { 1198 struct tcp6_sock *newtcp6sk = (struct tcp6_sock *)newsk; 1199 struct inet_request_sock *treq = inet_rsk(oreq); 1200 struct ipv6_pinfo *newnp = inet6_sk(newsk); 1201 struct ipv6_pinfo *np = inet6_sk(lsk); 1202 1203 inet_sk(newsk)->pinet6 = &newtcp6sk->inet6; 1204 memcpy(newnp, np, sizeof(struct ipv6_pinfo)); 1205 newsk->sk_v6_daddr = treq->ir_v6_rmt_addr; 1206 newsk->sk_v6_rcv_saddr = treq->ir_v6_loc_addr; 1207 inet6_sk(newsk)->saddr = treq->ir_v6_loc_addr; 1208 newnp->ipv6_fl_list = NULL; 1209 newnp->pktoptions = NULL; 1210 newsk->sk_bound_dev_if = treq->ir_iif; 1211 newinet->inet_opt = NULL; 1212 newinet->inet_daddr = LOOPBACK4_IPV6; 1213 newinet->inet_saddr = LOOPBACK4_IPV6; 1214 #endif 1215 } 1216 1217 oreq->ts_recent = PASS_OPEN_TID_G(ntohl(req->tos_stid)); 1218 sk_setup_caps(newsk, dst); 1219 ctx = tls_get_ctx(lsk); 1220 newsk->sk_destruct = ctx->sk_destruct; 1221 newsk->sk_prot_creator = lsk->sk_prot_creator; 1222 csk->sk = newsk; 1223 csk->passive_reap_next = oreq; 1224 csk->tx_chan = cxgb4_port_chan(ndev); 1225 csk->port_id = port_id; 1226 csk->egress_dev = ndev; 1227 csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid)); 1228 chtls_set_tcp_window(csk); 1229 tp->rcv_wnd = csk->rcv_win; 1230 csk->sndbuf = csk->snd_win; 1231 csk->ulp_mode = ULP_MODE_TLS; 1232 step = cdev->lldi->nrxq / cdev->lldi->nchan; 1233 rxq_idx = port_id * step; 1234 rxq_idx += cdev->round_robin_cnt++ % step; 1235 csk->rss_qid = cdev->lldi->rxq_ids[rxq_idx]; 1236 csk->txq_idx = (rxq_idx < cdev->lldi->ntxq) ? rxq_idx : 1237 port_id * step; 1238 csk->sndbuf = newsk->sk_sndbuf; 1239 csk->smac_idx = ((struct port_info *)netdev_priv(ndev))->smt_idx; 1240 RCV_WSCALE(tp) = select_rcv_wscale(tcp_full_space(newsk), 1241 sock_net(newsk)-> 1242 ipv4.sysctl_tcp_window_scaling, 1243 tp->window_clamp); 1244 neigh_release(n); 1245 inet_inherit_port(&tcp_hashinfo, lsk, newsk); 1246 csk_set_flag(csk, CSK_CONN_INLINE); 1247 bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */ 1248 1249 return newsk; 1250 free_csk: 1251 chtls_sock_release(&csk->kref); 1252 free_dst: 1253 neigh_release(n); 1254 dst_release(dst); 1255 free_sk: 1256 inet_csk_prepare_forced_close(newsk); 1257 tcp_done(newsk); 1258 free_oreq: 1259 chtls_reqsk_free(oreq); 1260 return NULL; 1261 } 1262 1263 /* 1264 * Populate a TID_RELEASE WR. The skb must be already propely sized. 1265 */ 1266 static void mk_tid_release(struct sk_buff *skb, 1267 unsigned int chan, unsigned int tid) 1268 { 1269 struct cpl_tid_release *req; 1270 unsigned int len; 1271 1272 len = roundup(sizeof(struct cpl_tid_release), 16); 1273 req = (struct cpl_tid_release *)__skb_put(skb, len); 1274 memset(req, 0, len); 1275 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan); 1276 INIT_TP_WR_CPL(req, CPL_TID_RELEASE, tid); 1277 } 1278 1279 static int chtls_get_module(struct sock *sk) 1280 { 1281 struct inet_connection_sock *icsk = inet_csk(sk); 1282 1283 if (!try_module_get(icsk->icsk_ulp_ops->owner)) 1284 return -1; 1285 1286 return 0; 1287 } 1288 1289 static void chtls_pass_accept_request(struct sock *sk, 1290 struct sk_buff *skb) 1291 { 1292 struct cpl_t5_pass_accept_rpl *rpl; 1293 struct cpl_pass_accept_req *req; 1294 struct listen_ctx *listen_ctx; 1295 struct vlan_ethhdr *vlan_eh; 1296 struct request_sock *oreq; 1297 struct sk_buff *reply_skb; 1298 struct chtls_sock *csk; 1299 struct chtls_dev *cdev; 1300 struct ipv6hdr *ip6h; 1301 struct tcphdr *tcph; 1302 struct sock *newsk; 1303 struct ethhdr *eh; 1304 struct iphdr *iph; 1305 void *network_hdr; 1306 unsigned int stid; 1307 unsigned int len; 1308 unsigned int tid; 1309 bool th_ecn, ect; 1310 __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */ 1311 u16 eth_hdr_len; 1312 bool ecn_ok; 1313 1314 req = cplhdr(skb) + RSS_HDR; 1315 tid = GET_TID(req); 1316 cdev = BLOG_SKB_CB(skb)->cdev; 1317 newsk = lookup_tid(cdev->tids, tid); 1318 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid)); 1319 if (newsk) { 1320 pr_info("tid (%d) already in use\n", tid); 1321 return; 1322 } 1323 1324 len = roundup(sizeof(*rpl), 16); 1325 reply_skb = alloc_skb(len, GFP_ATOMIC); 1326 if (!reply_skb) { 1327 cxgb4_remove_tid(cdev->tids, 0, tid, sk->sk_family); 1328 kfree_skb(skb); 1329 return; 1330 } 1331 1332 if (sk->sk_state != TCP_LISTEN) 1333 goto reject; 1334 1335 if (inet_csk_reqsk_queue_is_full(sk)) 1336 goto reject; 1337 1338 if (sk_acceptq_is_full(sk)) 1339 goto reject; 1340 1341 1342 eth_hdr_len = T6_ETH_HDR_LEN_G(ntohl(req->hdr_len)); 1343 if (eth_hdr_len == ETH_HLEN) { 1344 eh = (struct ethhdr *)(req + 1); 1345 iph = (struct iphdr *)(eh + 1); 1346 ip6h = (struct ipv6hdr *)(eh + 1); 1347 network_hdr = (void *)(eh + 1); 1348 } else { 1349 vlan_eh = (struct vlan_ethhdr *)(req + 1); 1350 iph = (struct iphdr *)(vlan_eh + 1); 1351 ip6h = (struct ipv6hdr *)(vlan_eh + 1); 1352 network_hdr = (void *)(vlan_eh + 1); 1353 } 1354 1355 if (iph->version == 0x4) { 1356 tcph = (struct tcphdr *)(iph + 1); 1357 skb_set_network_header(skb, (void *)iph - (void *)req); 1358 oreq = inet_reqsk_alloc(&chtls_rsk_ops, sk, true); 1359 } else { 1360 tcph = (struct tcphdr *)(ip6h + 1); 1361 skb_set_network_header(skb, (void *)ip6h - (void *)req); 1362 oreq = inet_reqsk_alloc(&chtls_rsk_opsv6, sk, false); 1363 } 1364 1365 if (!oreq) 1366 goto reject; 1367 1368 oreq->rsk_rcv_wnd = 0; 1369 oreq->rsk_window_clamp = 0; 1370 oreq->syncookie = 0; 1371 oreq->mss = 0; 1372 oreq->ts_recent = 0; 1373 1374 tcp_rsk(oreq)->tfo_listener = false; 1375 tcp_rsk(oreq)->rcv_isn = ntohl(tcph->seq); 1376 chtls_set_req_port(oreq, tcph->source, tcph->dest); 1377 if (iph->version == 0x4) { 1378 chtls_set_req_addr(oreq, iph->daddr, iph->saddr); 1379 ip_dsfield = ipv4_get_dsfield(iph); 1380 #if IS_ENABLED(CONFIG_IPV6) 1381 } else { 1382 inet_rsk(oreq)->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr; 1383 inet_rsk(oreq)->ir_v6_loc_addr = ipv6_hdr(skb)->daddr; 1384 ip_dsfield = ipv6_get_dsfield(ipv6_hdr(skb)); 1385 #endif 1386 } 1387 if (req->tcpopt.wsf <= 14 && 1388 sock_net(sk)->ipv4.sysctl_tcp_window_scaling) { 1389 inet_rsk(oreq)->wscale_ok = 1; 1390 inet_rsk(oreq)->snd_wscale = req->tcpopt.wsf; 1391 } 1392 inet_rsk(oreq)->ir_iif = sk->sk_bound_dev_if; 1393 th_ecn = tcph->ece && tcph->cwr; 1394 if (th_ecn) { 1395 ect = !INET_ECN_is_not_ect(ip_dsfield); 1396 ecn_ok = sock_net(sk)->ipv4.sysctl_tcp_ecn; 1397 if ((!ect && ecn_ok) || tcp_ca_needs_ecn(sk)) 1398 inet_rsk(oreq)->ecn_ok = 1; 1399 } 1400 1401 newsk = chtls_recv_sock(sk, oreq, network_hdr, req, cdev); 1402 if (!newsk) 1403 goto reject; 1404 1405 if (chtls_get_module(newsk)) 1406 goto reject; 1407 inet_csk_reqsk_queue_added(sk); 1408 reply_skb->sk = newsk; 1409 chtls_install_cpl_ops(newsk); 1410 cxgb4_insert_tid(cdev->tids, newsk, tid, newsk->sk_family); 1411 csk = rcu_dereference_sk_user_data(newsk); 1412 listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid); 1413 csk->listen_ctx = listen_ctx; 1414 __skb_queue_tail(&listen_ctx->synq, (struct sk_buff *)&csk->synq); 1415 chtls_pass_accept_rpl(reply_skb, req, tid); 1416 kfree_skb(skb); 1417 return; 1418 1419 reject: 1420 mk_tid_release(reply_skb, 0, tid); 1421 cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb); 1422 kfree_skb(skb); 1423 } 1424 1425 /* 1426 * Handle a CPL_PASS_ACCEPT_REQ message. 1427 */ 1428 static int chtls_pass_accept_req(struct chtls_dev *cdev, struct sk_buff *skb) 1429 { 1430 struct cpl_pass_accept_req *req = cplhdr(skb) + RSS_HDR; 1431 struct listen_ctx *ctx; 1432 unsigned int stid; 1433 unsigned int tid; 1434 struct sock *lsk; 1435 void *data; 1436 1437 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid)); 1438 tid = GET_TID(req); 1439 1440 data = lookup_stid(cdev->tids, stid); 1441 if (!data) 1442 return 1; 1443 1444 ctx = (struct listen_ctx *)data; 1445 lsk = ctx->lsk; 1446 1447 if (unlikely(tid_out_of_range(cdev->tids, tid))) { 1448 pr_info("passive open TID %u too large\n", tid); 1449 return 1; 1450 } 1451 1452 BLOG_SKB_CB(skb)->cdev = cdev; 1453 process_cpl_msg(chtls_pass_accept_request, lsk, skb); 1454 return 0; 1455 } 1456 1457 /* 1458 * Completes some final bits of initialization for just established connections 1459 * and changes their state to TCP_ESTABLISHED. 1460 * 1461 * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1. 1462 */ 1463 static void make_established(struct sock *sk, u32 snd_isn, unsigned int opt) 1464 { 1465 struct tcp_sock *tp = tcp_sk(sk); 1466 1467 tp->pushed_seq = snd_isn; 1468 tp->write_seq = snd_isn; 1469 tp->snd_nxt = snd_isn; 1470 tp->snd_una = snd_isn; 1471 inet_sk(sk)->inet_id = prandom_u32(); 1472 assign_rxopt(sk, opt); 1473 1474 if (tp->rcv_wnd > (RCV_BUFSIZ_M << 10)) 1475 tp->rcv_wup -= tp->rcv_wnd - (RCV_BUFSIZ_M << 10); 1476 1477 smp_mb(); 1478 tcp_set_state(sk, TCP_ESTABLISHED); 1479 } 1480 1481 static void chtls_abort_conn(struct sock *sk, struct sk_buff *skb) 1482 { 1483 struct sk_buff *abort_skb; 1484 1485 abort_skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC); 1486 if (abort_skb) 1487 chtls_send_reset(sk, CPL_ABORT_SEND_RST, abort_skb); 1488 } 1489 1490 static struct sock *reap_list; 1491 static DEFINE_SPINLOCK(reap_list_lock); 1492 1493 /* 1494 * Process the reap list. 1495 */ 1496 DECLARE_TASK_FUNC(process_reap_list, task_param) 1497 { 1498 spin_lock_bh(&reap_list_lock); 1499 while (reap_list) { 1500 struct sock *sk = reap_list; 1501 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); 1502 1503 reap_list = csk->passive_reap_next; 1504 csk->passive_reap_next = NULL; 1505 spin_unlock(&reap_list_lock); 1506 sock_hold(sk); 1507 1508 bh_lock_sock(sk); 1509 chtls_abort_conn(sk, NULL); 1510 sock_orphan(sk); 1511 if (sk->sk_state == TCP_CLOSE) 1512 inet_csk_destroy_sock(sk); 1513 bh_unlock_sock(sk); 1514 sock_put(sk); 1515 spin_lock(&reap_list_lock); 1516 } 1517 spin_unlock_bh(&reap_list_lock); 1518 } 1519 1520 static DECLARE_WORK(reap_task, process_reap_list); 1521 1522 static void add_to_reap_list(struct sock *sk) 1523 { 1524 struct chtls_sock *csk = sk->sk_user_data; 1525 1526 local_bh_disable(); 1527 release_tcp_port(sk); /* release the port immediately */ 1528 1529 spin_lock(&reap_list_lock); 1530 csk->passive_reap_next = reap_list; 1531 reap_list = sk; 1532 if (!csk->passive_reap_next) 1533 schedule_work(&reap_task); 1534 spin_unlock(&reap_list_lock); 1535 local_bh_enable(); 1536 } 1537 1538 static void add_pass_open_to_parent(struct sock *child, struct sock *lsk, 1539 struct chtls_dev *cdev) 1540 { 1541 struct request_sock *oreq; 1542 struct chtls_sock *csk; 1543 1544 if (lsk->sk_state != TCP_LISTEN) 1545 return; 1546 1547 csk = child->sk_user_data; 1548 oreq = csk->passive_reap_next; 1549 csk->passive_reap_next = NULL; 1550 1551 reqsk_queue_removed(&inet_csk(lsk)->icsk_accept_queue, oreq); 1552 __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq); 1553 1554 if (sk_acceptq_is_full(lsk)) { 1555 chtls_reqsk_free(oreq); 1556 add_to_reap_list(child); 1557 } else { 1558 refcount_set(&oreq->rsk_refcnt, 1); 1559 inet_csk_reqsk_queue_add(lsk, oreq, child); 1560 lsk->sk_data_ready(lsk); 1561 } 1562 } 1563 1564 static void bl_add_pass_open_to_parent(struct sock *lsk, struct sk_buff *skb) 1565 { 1566 struct sock *child = skb->sk; 1567 1568 skb->sk = NULL; 1569 add_pass_open_to_parent(child, lsk, BLOG_SKB_CB(skb)->cdev); 1570 kfree_skb(skb); 1571 } 1572 1573 static int chtls_pass_establish(struct chtls_dev *cdev, struct sk_buff *skb) 1574 { 1575 struct cpl_pass_establish *req = cplhdr(skb) + RSS_HDR; 1576 struct chtls_sock *csk; 1577 struct sock *lsk, *sk; 1578 unsigned int hwtid; 1579 1580 hwtid = GET_TID(req); 1581 sk = lookup_tid(cdev->tids, hwtid); 1582 if (!sk) 1583 return (CPL_RET_UNKNOWN_TID | CPL_RET_BUF_DONE); 1584 1585 bh_lock_sock(sk); 1586 if (unlikely(sock_owned_by_user(sk))) { 1587 kfree_skb(skb); 1588 } else { 1589 unsigned int stid; 1590 void *data; 1591 1592 csk = sk->sk_user_data; 1593 csk->wr_max_credits = 64; 1594 csk->wr_credits = 64; 1595 csk->wr_unacked = 0; 1596 make_established(sk, ntohl(req->snd_isn), ntohs(req->tcp_opt)); 1597 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid)); 1598 sk->sk_state_change(sk); 1599 if (unlikely(sk->sk_socket)) 1600 sk_wake_async(sk, 0, POLL_OUT); 1601 1602 data = lookup_stid(cdev->tids, stid); 1603 if (!data) { 1604 /* listening server close */ 1605 kfree_skb(skb); 1606 goto unlock; 1607 } 1608 lsk = ((struct listen_ctx *)data)->lsk; 1609 1610 bh_lock_sock(lsk); 1611 if (unlikely(skb_queue_empty(&csk->listen_ctx->synq))) { 1612 /* removed from synq */ 1613 bh_unlock_sock(lsk); 1614 kfree_skb(skb); 1615 goto unlock; 1616 } 1617 1618 if (likely(!sock_owned_by_user(lsk))) { 1619 kfree_skb(skb); 1620 add_pass_open_to_parent(sk, lsk, cdev); 1621 } else { 1622 skb->sk = sk; 1623 BLOG_SKB_CB(skb)->cdev = cdev; 1624 BLOG_SKB_CB(skb)->backlog_rcv = 1625 bl_add_pass_open_to_parent; 1626 __sk_add_backlog(lsk, skb); 1627 } 1628 bh_unlock_sock(lsk); 1629 } 1630 unlock: 1631 bh_unlock_sock(sk); 1632 return 0; 1633 } 1634 1635 /* 1636 * Handle receipt of an urgent pointer. 1637 */ 1638 static void handle_urg_ptr(struct sock *sk, u32 urg_seq) 1639 { 1640 struct tcp_sock *tp = tcp_sk(sk); 1641 1642 urg_seq--; 1643 if (tp->urg_data && !after(urg_seq, tp->urg_seq)) 1644 return; /* duplicate pointer */ 1645 1646 sk_send_sigurg(sk); 1647 if (tp->urg_seq == tp->copied_seq && tp->urg_data && 1648 !sock_flag(sk, SOCK_URGINLINE) && 1649 tp->copied_seq != tp->rcv_nxt) { 1650 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); 1651 1652 tp->copied_seq++; 1653 if (skb && tp->copied_seq - ULP_SKB_CB(skb)->seq >= skb->len) 1654 chtls_free_skb(sk, skb); 1655 } 1656 1657 tp->urg_data = TCP_URG_NOTYET; 1658 tp->urg_seq = urg_seq; 1659 } 1660 1661 static void check_sk_callbacks(struct chtls_sock *csk) 1662 { 1663 struct sock *sk = csk->sk; 1664 1665 if (unlikely(sk->sk_user_data && 1666 !csk_flag_nochk(csk, CSK_CALLBACKS_CHKD))) 1667 csk_set_flag(csk, CSK_CALLBACKS_CHKD); 1668 } 1669 1670 /* 1671 * Handles Rx data that arrives in a state where the socket isn't accepting 1672 * new data. 1673 */ 1674 static void handle_excess_rx(struct sock *sk, struct sk_buff *skb) 1675 { 1676 if (!csk_flag(sk, CSK_ABORT_SHUTDOWN)) 1677 chtls_abort_conn(sk, skb); 1678 1679 kfree_skb(skb); 1680 } 1681 1682 static void chtls_recv_data(struct sock *sk, struct sk_buff *skb) 1683 { 1684 struct cpl_rx_data *hdr = cplhdr(skb) + RSS_HDR; 1685 struct chtls_sock *csk; 1686 struct tcp_sock *tp; 1687 1688 csk = rcu_dereference_sk_user_data(sk); 1689 tp = tcp_sk(sk); 1690 1691 if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) { 1692 handle_excess_rx(sk, skb); 1693 return; 1694 } 1695 1696 ULP_SKB_CB(skb)->seq = ntohl(hdr->seq); 1697 ULP_SKB_CB(skb)->psh = hdr->psh; 1698 skb_ulp_mode(skb) = ULP_MODE_NONE; 1699 1700 skb_reset_transport_header(skb); 1701 __skb_pull(skb, sizeof(*hdr) + RSS_HDR); 1702 if (!skb->data_len) 1703 __skb_trim(skb, ntohs(hdr->len)); 1704 1705 if (unlikely(hdr->urg)) 1706 handle_urg_ptr(sk, tp->rcv_nxt + ntohs(hdr->urg)); 1707 if (unlikely(tp->urg_data == TCP_URG_NOTYET && 1708 tp->urg_seq - tp->rcv_nxt < skb->len)) 1709 tp->urg_data = TCP_URG_VALID | 1710 skb->data[tp->urg_seq - tp->rcv_nxt]; 1711 1712 if (unlikely(hdr->dack_mode != csk->delack_mode)) { 1713 csk->delack_mode = hdr->dack_mode; 1714 csk->delack_seq = tp->rcv_nxt; 1715 } 1716 1717 tcp_hdr(skb)->fin = 0; 1718 tp->rcv_nxt += skb->len; 1719 1720 __skb_queue_tail(&sk->sk_receive_queue, skb); 1721 1722 if (!sock_flag(sk, SOCK_DEAD)) { 1723 check_sk_callbacks(csk); 1724 sk->sk_data_ready(sk); 1725 } 1726 } 1727 1728 static int chtls_rx_data(struct chtls_dev *cdev, struct sk_buff *skb) 1729 { 1730 struct cpl_rx_data *req = cplhdr(skb) + RSS_HDR; 1731 unsigned int hwtid = GET_TID(req); 1732 struct sock *sk; 1733 1734 sk = lookup_tid(cdev->tids, hwtid); 1735 if (unlikely(!sk)) { 1736 pr_err("can't find conn. for hwtid %u.\n", hwtid); 1737 return -EINVAL; 1738 } 1739 skb_dst_set(skb, NULL); 1740 process_cpl_msg(chtls_recv_data, sk, skb); 1741 return 0; 1742 } 1743 1744 static void chtls_recv_pdu(struct sock *sk, struct sk_buff *skb) 1745 { 1746 struct cpl_tls_data *hdr = cplhdr(skb); 1747 struct chtls_sock *csk; 1748 struct chtls_hws *tlsk; 1749 struct tcp_sock *tp; 1750 1751 csk = rcu_dereference_sk_user_data(sk); 1752 tlsk = &csk->tlshws; 1753 tp = tcp_sk(sk); 1754 1755 if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) { 1756 handle_excess_rx(sk, skb); 1757 return; 1758 } 1759 1760 ULP_SKB_CB(skb)->seq = ntohl(hdr->seq); 1761 ULP_SKB_CB(skb)->flags = 0; 1762 skb_ulp_mode(skb) = ULP_MODE_TLS; 1763 1764 skb_reset_transport_header(skb); 1765 __skb_pull(skb, sizeof(*hdr)); 1766 if (!skb->data_len) 1767 __skb_trim(skb, 1768 CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd))); 1769 1770 if (unlikely(tp->urg_data == TCP_URG_NOTYET && tp->urg_seq - 1771 tp->rcv_nxt < skb->len)) 1772 tp->urg_data = TCP_URG_VALID | 1773 skb->data[tp->urg_seq - tp->rcv_nxt]; 1774 1775 tcp_hdr(skb)->fin = 0; 1776 tlsk->pldlen = CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd)); 1777 __skb_queue_tail(&tlsk->sk_recv_queue, skb); 1778 } 1779 1780 static int chtls_rx_pdu(struct chtls_dev *cdev, struct sk_buff *skb) 1781 { 1782 struct cpl_tls_data *req = cplhdr(skb); 1783 unsigned int hwtid = GET_TID(req); 1784 struct sock *sk; 1785 1786 sk = lookup_tid(cdev->tids, hwtid); 1787 if (unlikely(!sk)) { 1788 pr_err("can't find conn. for hwtid %u.\n", hwtid); 1789 return -EINVAL; 1790 } 1791 skb_dst_set(skb, NULL); 1792 process_cpl_msg(chtls_recv_pdu, sk, skb); 1793 return 0; 1794 } 1795 1796 static void chtls_set_hdrlen(struct sk_buff *skb, unsigned int nlen) 1797 { 1798 struct tlsrx_cmp_hdr *tls_cmp_hdr = cplhdr(skb); 1799 1800 skb->hdr_len = ntohs((__force __be16)tls_cmp_hdr->length); 1801 tls_cmp_hdr->length = ntohs((__force __be16)nlen); 1802 } 1803 1804 static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb) 1805 { 1806 struct tlsrx_cmp_hdr *tls_hdr_pkt; 1807 struct cpl_rx_tls_cmp *cmp_cpl; 1808 struct sk_buff *skb_rec; 1809 struct chtls_sock *csk; 1810 struct chtls_hws *tlsk; 1811 struct tcp_sock *tp; 1812 1813 cmp_cpl = cplhdr(skb); 1814 csk = rcu_dereference_sk_user_data(sk); 1815 tlsk = &csk->tlshws; 1816 tp = tcp_sk(sk); 1817 1818 ULP_SKB_CB(skb)->seq = ntohl(cmp_cpl->seq); 1819 ULP_SKB_CB(skb)->flags = 0; 1820 1821 skb_reset_transport_header(skb); 1822 __skb_pull(skb, sizeof(*cmp_cpl)); 1823 tls_hdr_pkt = (struct tlsrx_cmp_hdr *)skb->data; 1824 if (tls_hdr_pkt->res_to_mac_error & TLSRX_HDR_PKT_ERROR_M) 1825 tls_hdr_pkt->type = CONTENT_TYPE_ERROR; 1826 if (!skb->data_len) 1827 __skb_trim(skb, TLS_HEADER_LENGTH); 1828 1829 tp->rcv_nxt += 1830 CPL_RX_TLS_CMP_PDULENGTH_G(ntohl(cmp_cpl->pdulength_length)); 1831 1832 ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_TLS_HDR; 1833 skb_rec = __skb_dequeue(&tlsk->sk_recv_queue); 1834 if (!skb_rec) { 1835 __skb_queue_tail(&sk->sk_receive_queue, skb); 1836 } else { 1837 chtls_set_hdrlen(skb, tlsk->pldlen); 1838 tlsk->pldlen = 0; 1839 __skb_queue_tail(&sk->sk_receive_queue, skb); 1840 __skb_queue_tail(&sk->sk_receive_queue, skb_rec); 1841 } 1842 1843 if (!sock_flag(sk, SOCK_DEAD)) { 1844 check_sk_callbacks(csk); 1845 sk->sk_data_ready(sk); 1846 } 1847 } 1848 1849 static int chtls_rx_cmp(struct chtls_dev *cdev, struct sk_buff *skb) 1850 { 1851 struct cpl_rx_tls_cmp *req = cplhdr(skb); 1852 unsigned int hwtid = GET_TID(req); 1853 struct sock *sk; 1854 1855 sk = lookup_tid(cdev->tids, hwtid); 1856 if (unlikely(!sk)) { 1857 pr_err("can't find conn. for hwtid %u.\n", hwtid); 1858 return -EINVAL; 1859 } 1860 skb_dst_set(skb, NULL); 1861 process_cpl_msg(chtls_rx_hdr, sk, skb); 1862 1863 return 0; 1864 } 1865 1866 static void chtls_timewait(struct sock *sk) 1867 { 1868 struct tcp_sock *tp = tcp_sk(sk); 1869 1870 tp->rcv_nxt++; 1871 tp->rx_opt.ts_recent_stamp = ktime_get_seconds(); 1872 tp->srtt_us = 0; 1873 tcp_time_wait(sk, TCP_TIME_WAIT, 0); 1874 } 1875 1876 static void chtls_peer_close(struct sock *sk, struct sk_buff *skb) 1877 { 1878 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); 1879 1880 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) 1881 goto out; 1882 1883 sk->sk_shutdown |= RCV_SHUTDOWN; 1884 sock_set_flag(sk, SOCK_DONE); 1885 1886 switch (sk->sk_state) { 1887 case TCP_SYN_RECV: 1888 case TCP_ESTABLISHED: 1889 tcp_set_state(sk, TCP_CLOSE_WAIT); 1890 break; 1891 case TCP_FIN_WAIT1: 1892 tcp_set_state(sk, TCP_CLOSING); 1893 break; 1894 case TCP_FIN_WAIT2: 1895 chtls_release_resources(sk); 1896 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) 1897 chtls_conn_done(sk); 1898 else 1899 chtls_timewait(sk); 1900 break; 1901 default: 1902 pr_info("cpl_peer_close in bad state %d\n", sk->sk_state); 1903 } 1904 1905 if (!sock_flag(sk, SOCK_DEAD)) { 1906 sk->sk_state_change(sk); 1907 /* Do not send POLL_HUP for half duplex close. */ 1908 1909 if ((sk->sk_shutdown & SEND_SHUTDOWN) || 1910 sk->sk_state == TCP_CLOSE) 1911 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); 1912 else 1913 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 1914 } 1915 out: 1916 kfree_skb(skb); 1917 } 1918 1919 static void chtls_close_con_rpl(struct sock *sk, struct sk_buff *skb) 1920 { 1921 struct cpl_close_con_rpl *rpl = cplhdr(skb) + RSS_HDR; 1922 struct chtls_sock *csk; 1923 struct tcp_sock *tp; 1924 1925 csk = rcu_dereference_sk_user_data(sk); 1926 1927 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) 1928 goto out; 1929 1930 tp = tcp_sk(sk); 1931 1932 tp->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */ 1933 1934 switch (sk->sk_state) { 1935 case TCP_CLOSING: 1936 chtls_release_resources(sk); 1937 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) 1938 chtls_conn_done(sk); 1939 else 1940 chtls_timewait(sk); 1941 break; 1942 case TCP_LAST_ACK: 1943 chtls_release_resources(sk); 1944 chtls_conn_done(sk); 1945 break; 1946 case TCP_FIN_WAIT1: 1947 tcp_set_state(sk, TCP_FIN_WAIT2); 1948 sk->sk_shutdown |= SEND_SHUTDOWN; 1949 1950 if (!sock_flag(sk, SOCK_DEAD)) 1951 sk->sk_state_change(sk); 1952 else if (tcp_sk(sk)->linger2 < 0 && 1953 !csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN)) 1954 chtls_abort_conn(sk, skb); 1955 else if (csk_flag_nochk(csk, CSK_TX_DATA_SENT)) 1956 chtls_set_quiesce_ctrl(sk, 0); 1957 break; 1958 default: 1959 pr_info("close_con_rpl in bad state %d\n", sk->sk_state); 1960 } 1961 out: 1962 kfree_skb(skb); 1963 } 1964 1965 static struct sk_buff *get_cpl_skb(struct sk_buff *skb, 1966 size_t len, gfp_t gfp) 1967 { 1968 if (likely(!skb_is_nonlinear(skb) && !skb_cloned(skb))) { 1969 WARN_ONCE(skb->len < len, "skb alloc error"); 1970 __skb_trim(skb, len); 1971 skb_get(skb); 1972 } else { 1973 skb = alloc_skb(len, gfp); 1974 if (skb) 1975 __skb_put(skb, len); 1976 } 1977 return skb; 1978 } 1979 1980 static void set_abort_rpl_wr(struct sk_buff *skb, unsigned int tid, 1981 int cmd) 1982 { 1983 struct cpl_abort_rpl *rpl = cplhdr(skb); 1984 1985 INIT_TP_WR_CPL(rpl, CPL_ABORT_RPL, tid); 1986 rpl->cmd = cmd; 1987 } 1988 1989 static void send_defer_abort_rpl(struct chtls_dev *cdev, struct sk_buff *skb) 1990 { 1991 struct cpl_abort_req_rss *req = cplhdr(skb); 1992 struct sk_buff *reply_skb; 1993 1994 reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl), 1995 GFP_KERNEL | __GFP_NOFAIL); 1996 __skb_put(reply_skb, sizeof(struct cpl_abort_rpl)); 1997 set_abort_rpl_wr(reply_skb, GET_TID(req), 1998 (req->status & CPL_ABORT_NO_RST)); 1999 set_wr_txq(reply_skb, CPL_PRIORITY_DATA, req->status >> 1); 2000 cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb); 2001 kfree_skb(skb); 2002 } 2003 2004 /* 2005 * Add an skb to the deferred skb queue for processing from process context. 2006 */ 2007 static void t4_defer_reply(struct sk_buff *skb, struct chtls_dev *cdev, 2008 defer_handler_t handler) 2009 { 2010 DEFERRED_SKB_CB(skb)->handler = handler; 2011 spin_lock_bh(&cdev->deferq.lock); 2012 __skb_queue_tail(&cdev->deferq, skb); 2013 if (skb_queue_len(&cdev->deferq) == 1) 2014 schedule_work(&cdev->deferq_task); 2015 spin_unlock_bh(&cdev->deferq.lock); 2016 } 2017 2018 static void chtls_send_abort_rpl(struct sock *sk, struct sk_buff *skb, 2019 struct chtls_dev *cdev, 2020 int status, int queue) 2021 { 2022 struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR; 2023 struct sk_buff *reply_skb; 2024 struct chtls_sock *csk; 2025 unsigned int tid; 2026 2027 csk = rcu_dereference_sk_user_data(sk); 2028 tid = GET_TID(req); 2029 2030 reply_skb = get_cpl_skb(skb, sizeof(struct cpl_abort_rpl), gfp_any()); 2031 if (!reply_skb) { 2032 req->status = (queue << 1) | status; 2033 t4_defer_reply(skb, cdev, send_defer_abort_rpl); 2034 return; 2035 } 2036 2037 set_abort_rpl_wr(reply_skb, tid, status); 2038 kfree_skb(skb); 2039 set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue); 2040 if (csk_conn_inline(csk)) { 2041 struct l2t_entry *e = csk->l2t_entry; 2042 2043 if (e && sk->sk_state != TCP_SYN_RECV) { 2044 cxgb4_l2t_send(csk->egress_dev, reply_skb, e); 2045 return; 2046 } 2047 } 2048 cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb); 2049 } 2050 2051 /* 2052 * This is run from a listener's backlog to abort a child connection in 2053 * SYN_RCV state (i.e., one on the listener's SYN queue). 2054 */ 2055 static void bl_abort_syn_rcv(struct sock *lsk, struct sk_buff *skb) 2056 { 2057 struct chtls_sock *csk; 2058 struct sock *child; 2059 int queue; 2060 2061 child = skb->sk; 2062 csk = rcu_dereference_sk_user_data(child); 2063 queue = csk->txq_idx; 2064 2065 skb->sk = NULL; 2066 chtls_send_abort_rpl(child, skb, BLOG_SKB_CB(skb)->cdev, 2067 CPL_ABORT_NO_RST, queue); 2068 do_abort_syn_rcv(child, lsk); 2069 } 2070 2071 static int abort_syn_rcv(struct sock *sk, struct sk_buff *skb) 2072 { 2073 const struct request_sock *oreq; 2074 struct listen_ctx *listen_ctx; 2075 struct chtls_sock *csk; 2076 struct chtls_dev *cdev; 2077 struct sock *psk; 2078 void *ctx; 2079 2080 csk = sk->sk_user_data; 2081 oreq = csk->passive_reap_next; 2082 cdev = csk->cdev; 2083 2084 if (!oreq) 2085 return -1; 2086 2087 ctx = lookup_stid(cdev->tids, oreq->ts_recent); 2088 if (!ctx) 2089 return -1; 2090 2091 listen_ctx = (struct listen_ctx *)ctx; 2092 psk = listen_ctx->lsk; 2093 2094 bh_lock_sock(psk); 2095 if (!sock_owned_by_user(psk)) { 2096 int queue = csk->txq_idx; 2097 2098 chtls_send_abort_rpl(sk, skb, cdev, CPL_ABORT_NO_RST, queue); 2099 do_abort_syn_rcv(sk, psk); 2100 } else { 2101 skb->sk = sk; 2102 BLOG_SKB_CB(skb)->backlog_rcv = bl_abort_syn_rcv; 2103 __sk_add_backlog(psk, skb); 2104 } 2105 bh_unlock_sock(psk); 2106 return 0; 2107 } 2108 2109 static void chtls_abort_req_rss(struct sock *sk, struct sk_buff *skb) 2110 { 2111 const struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR; 2112 struct chtls_sock *csk = sk->sk_user_data; 2113 int rst_status = CPL_ABORT_NO_RST; 2114 int queue = csk->txq_idx; 2115 2116 if (is_neg_adv(req->status)) { 2117 kfree_skb(skb); 2118 return; 2119 } 2120 2121 csk_reset_flag(csk, CSK_ABORT_REQ_RCVD); 2122 2123 if (!csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) && 2124 !csk_flag_nochk(csk, CSK_TX_DATA_SENT)) { 2125 struct tcp_sock *tp = tcp_sk(sk); 2126 2127 if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0) 2128 WARN_ONCE(1, "send_tx_flowc error"); 2129 csk_set_flag(csk, CSK_TX_DATA_SENT); 2130 } 2131 2132 csk_set_flag(csk, CSK_ABORT_SHUTDOWN); 2133 2134 if (!csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) { 2135 sk->sk_err = ETIMEDOUT; 2136 2137 if (!sock_flag(sk, SOCK_DEAD)) 2138 sk->sk_error_report(sk); 2139 2140 if (sk->sk_state == TCP_SYN_RECV && !abort_syn_rcv(sk, skb)) 2141 return; 2142 2143 } 2144 2145 chtls_send_abort_rpl(sk, skb, BLOG_SKB_CB(skb)->cdev, 2146 rst_status, queue); 2147 chtls_release_resources(sk); 2148 chtls_conn_done(sk); 2149 } 2150 2151 static void chtls_abort_rpl_rss(struct sock *sk, struct sk_buff *skb) 2152 { 2153 struct cpl_abort_rpl_rss *rpl = cplhdr(skb) + RSS_HDR; 2154 struct chtls_sock *csk; 2155 struct chtls_dev *cdev; 2156 2157 csk = rcu_dereference_sk_user_data(sk); 2158 cdev = csk->cdev; 2159 2160 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) { 2161 csk_reset_flag(csk, CSK_ABORT_RPL_PENDING); 2162 if (!csk_flag_nochk(csk, CSK_ABORT_REQ_RCVD)) { 2163 if (sk->sk_state == TCP_SYN_SENT) { 2164 cxgb4_remove_tid(cdev->tids, 2165 csk->port_id, 2166 GET_TID(rpl), 2167 sk->sk_family); 2168 sock_put(sk); 2169 } 2170 chtls_release_resources(sk); 2171 chtls_conn_done(sk); 2172 } 2173 } 2174 kfree_skb(skb); 2175 } 2176 2177 static int chtls_conn_cpl(struct chtls_dev *cdev, struct sk_buff *skb) 2178 { 2179 struct cpl_peer_close *req = cplhdr(skb) + RSS_HDR; 2180 void (*fn)(struct sock *sk, struct sk_buff *skb); 2181 unsigned int hwtid = GET_TID(req); 2182 struct chtls_sock *csk; 2183 struct sock *sk; 2184 u8 opcode; 2185 2186 opcode = ((const struct rss_header *)cplhdr(skb))->opcode; 2187 2188 sk = lookup_tid(cdev->tids, hwtid); 2189 if (!sk) 2190 goto rel_skb; 2191 2192 csk = sk->sk_user_data; 2193 2194 switch (opcode) { 2195 case CPL_PEER_CLOSE: 2196 fn = chtls_peer_close; 2197 break; 2198 case CPL_CLOSE_CON_RPL: 2199 fn = chtls_close_con_rpl; 2200 break; 2201 case CPL_ABORT_REQ_RSS: 2202 /* 2203 * Save the offload device in the skb, we may process this 2204 * message after the socket has closed. 2205 */ 2206 BLOG_SKB_CB(skb)->cdev = csk->cdev; 2207 fn = chtls_abort_req_rss; 2208 break; 2209 case CPL_ABORT_RPL_RSS: 2210 fn = chtls_abort_rpl_rss; 2211 break; 2212 default: 2213 goto rel_skb; 2214 } 2215 2216 process_cpl_msg(fn, sk, skb); 2217 return 0; 2218 2219 rel_skb: 2220 kfree_skb(skb); 2221 return 0; 2222 } 2223 2224 static void chtls_rx_ack(struct sock *sk, struct sk_buff *skb) 2225 { 2226 struct cpl_fw4_ack *hdr = cplhdr(skb) + RSS_HDR; 2227 struct chtls_sock *csk = sk->sk_user_data; 2228 struct tcp_sock *tp = tcp_sk(sk); 2229 u32 credits = hdr->credits; 2230 u32 snd_una; 2231 2232 snd_una = ntohl(hdr->snd_una); 2233 csk->wr_credits += credits; 2234 2235 if (csk->wr_unacked > csk->wr_max_credits - csk->wr_credits) 2236 csk->wr_unacked = csk->wr_max_credits - csk->wr_credits; 2237 2238 while (credits) { 2239 struct sk_buff *pskb = csk->wr_skb_head; 2240 u32 csum; 2241 2242 if (unlikely(!pskb)) { 2243 if (csk->wr_nondata) 2244 csk->wr_nondata -= credits; 2245 break; 2246 } 2247 csum = (__force u32)pskb->csum; 2248 if (unlikely(credits < csum)) { 2249 pskb->csum = (__force __wsum)(csum - credits); 2250 break; 2251 } 2252 dequeue_wr(sk); 2253 credits -= csum; 2254 kfree_skb(pskb); 2255 } 2256 if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) { 2257 if (unlikely(before(snd_una, tp->snd_una))) { 2258 kfree_skb(skb); 2259 return; 2260 } 2261 2262 if (tp->snd_una != snd_una) { 2263 tp->snd_una = snd_una; 2264 tp->rcv_tstamp = tcp_time_stamp(tp); 2265 if (tp->snd_una == tp->snd_nxt && 2266 !csk_flag_nochk(csk, CSK_TX_FAILOVER)) 2267 csk_reset_flag(csk, CSK_TX_WAIT_IDLE); 2268 } 2269 } 2270 2271 if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_CH) { 2272 unsigned int fclen16 = roundup(failover_flowc_wr_len, 16); 2273 2274 csk->wr_credits -= fclen16; 2275 csk_reset_flag(csk, CSK_TX_WAIT_IDLE); 2276 csk_reset_flag(csk, CSK_TX_FAILOVER); 2277 } 2278 if (skb_queue_len(&csk->txq) && chtls_push_frames(csk, 0)) 2279 sk->sk_write_space(sk); 2280 2281 kfree_skb(skb); 2282 } 2283 2284 static int chtls_wr_ack(struct chtls_dev *cdev, struct sk_buff *skb) 2285 { 2286 struct cpl_fw4_ack *rpl = cplhdr(skb) + RSS_HDR; 2287 unsigned int hwtid = GET_TID(rpl); 2288 struct sock *sk; 2289 2290 sk = lookup_tid(cdev->tids, hwtid); 2291 if (unlikely(!sk)) { 2292 pr_err("can't find conn. for hwtid %u.\n", hwtid); 2293 return -EINVAL; 2294 } 2295 process_cpl_msg(chtls_rx_ack, sk, skb); 2296 2297 return 0; 2298 } 2299 2300 static int chtls_set_tcb_rpl(struct chtls_dev *cdev, struct sk_buff *skb) 2301 { 2302 struct cpl_set_tcb_rpl *rpl = cplhdr(skb) + RSS_HDR; 2303 unsigned int hwtid = GET_TID(rpl); 2304 struct sock *sk; 2305 2306 sk = lookup_tid(cdev->tids, hwtid); 2307 2308 /* return EINVAL if socket doesn't exist */ 2309 if (!sk) 2310 return -EINVAL; 2311 2312 /* Reusing the skb as size of cpl_set_tcb_field structure 2313 * is greater than cpl_abort_req 2314 */ 2315 if (TCB_COOKIE_G(rpl->cookie) == TCB_FIELD_COOKIE_TFLAG) 2316 chtls_send_abort(sk, CPL_ABORT_SEND_RST, NULL); 2317 2318 kfree_skb(skb); 2319 return 0; 2320 } 2321 2322 chtls_handler_func chtls_handlers[NUM_CPL_CMDS] = { 2323 [CPL_PASS_OPEN_RPL] = chtls_pass_open_rpl, 2324 [CPL_CLOSE_LISTSRV_RPL] = chtls_close_listsrv_rpl, 2325 [CPL_PASS_ACCEPT_REQ] = chtls_pass_accept_req, 2326 [CPL_PASS_ESTABLISH] = chtls_pass_establish, 2327 [CPL_RX_DATA] = chtls_rx_data, 2328 [CPL_TLS_DATA] = chtls_rx_pdu, 2329 [CPL_RX_TLS_CMP] = chtls_rx_cmp, 2330 [CPL_PEER_CLOSE] = chtls_conn_cpl, 2331 [CPL_CLOSE_CON_RPL] = chtls_conn_cpl, 2332 [CPL_ABORT_REQ_RSS] = chtls_conn_cpl, 2333 [CPL_ABORT_RPL_RSS] = chtls_conn_cpl, 2334 [CPL_FW4_ACK] = chtls_wr_ack, 2335 [CPL_SET_TCB_RPL] = chtls_set_tcb_rpl, 2336 }; 2337