1 /* 2 * Shared Memory Communications over RDMA (SMC-R) and RoCE 3 * 4 * AF_SMC protocol family socket handler keeping the AF_INET sock address type 5 * applies to SOCK_STREAM sockets only 6 * offers an alternative communication option for TCP-protocol sockets 7 * applicable with RoCE-cards only 8 * 9 * Initial restrictions: 10 * - non-blocking connect postponed 11 * - IPv6 support postponed 12 * - support for alternate links postponed 13 * - partial support for non-blocking sockets only 14 * - support for urgent data postponed 15 * 16 * Copyright IBM Corp. 2016 17 * 18 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> 19 * based on prototype from Frank Blaschka 20 */ 21 22 #define KMSG_COMPONENT "smc" 23 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 24 25 #include <linux/module.h> 26 #include <linux/socket.h> 27 #include <linux/inetdevice.h> 28 #include <linux/workqueue.h> 29 #include <linux/in.h> 30 #include <linux/sched/signal.h> 31 32 #include <net/sock.h> 33 #include <net/tcp.h> 34 #include <net/smc.h> 35 36 #include "smc.h" 37 #include "smc_clc.h" 38 #include "smc_llc.h" 39 #include "smc_cdc.h" 40 #include "smc_core.h" 41 #include "smc_ib.h" 42 #include "smc_pnet.h" 43 #include "smc_tx.h" 44 #include "smc_rx.h" 45 #include "smc_close.h" 46 47 static DEFINE_MUTEX(smc_create_lgr_pending); /* serialize link group 48 * creation 49 */ 50 51 struct smc_lgr_list smc_lgr_list = { /* established link groups */ 52 .lock = __SPIN_LOCK_UNLOCKED(smc_lgr_list.lock), 53 .list = LIST_HEAD_INIT(smc_lgr_list.list), 54 }; 55 56 static void smc_tcp_listen_work(struct work_struct *); 57 58 static void smc_set_keepalive(struct sock *sk, int val) 59 { 60 struct smc_sock *smc = smc_sk(sk); 61 62 smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val); 63 } 64 65 static struct smc_hashinfo smc_v4_hashinfo = { 66 .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock), 67 }; 68 69 int smc_hash_sk(struct sock *sk) 70 { 71 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash; 72 struct hlist_head *head; 73 74 head = &h->ht; 75 76 write_lock_bh(&h->lock); 77 sk_add_node(sk, head); 78 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 79 write_unlock_bh(&h->lock); 80 81 return 0; 82 } 83 EXPORT_SYMBOL_GPL(smc_hash_sk); 84 85 void smc_unhash_sk(struct sock *sk) 86 { 87 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash; 88 89 write_lock_bh(&h->lock); 90 if (sk_del_node_init(sk)) 91 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); 92 write_unlock_bh(&h->lock); 93 } 94 EXPORT_SYMBOL_GPL(smc_unhash_sk); 95 96 struct proto smc_proto = { 97 .name = "SMC", 98 .owner = THIS_MODULE, 99 .keepalive = smc_set_keepalive, 100 .hash = smc_hash_sk, 101 .unhash = smc_unhash_sk, 102 .obj_size = sizeof(struct smc_sock), 103 .h.smc_hash = &smc_v4_hashinfo, 104 .slab_flags = SLAB_TYPESAFE_BY_RCU, 105 }; 106 EXPORT_SYMBOL_GPL(smc_proto); 107 108 static int smc_release(struct socket *sock) 109 { 110 struct sock *sk = sock->sk; 111 struct smc_sock *smc; 112 int rc = 0; 113 114 if (!sk) 115 goto out; 116 117 smc = smc_sk(sk); 118 sock_hold(sk); 119 if (sk->sk_state == SMC_LISTEN) 120 /* smc_close_non_accepted() is called and acquires 121 * sock lock for child sockets again 122 */ 123 lock_sock_nested(sk, SINGLE_DEPTH_NESTING); 124 else 125 lock_sock(sk); 126 127 if (smc->use_fallback) { 128 sk->sk_state = SMC_CLOSED; 129 sk->sk_state_change(sk); 130 } else { 131 rc = smc_close_active(smc); 132 sock_set_flag(sk, SOCK_DEAD); 133 sk->sk_shutdown |= SHUTDOWN_MASK; 134 } 135 if (smc->clcsock) { 136 sock_release(smc->clcsock); 137 smc->clcsock = NULL; 138 } 139 140 /* detach socket */ 141 sock_orphan(sk); 142 sock->sk = NULL; 143 if (smc->use_fallback) { 144 schedule_delayed_work(&smc->sock_put_work, TCP_TIMEWAIT_LEN); 145 } else if (sk->sk_state == SMC_CLOSED) { 146 smc_conn_free(&smc->conn); 147 schedule_delayed_work(&smc->sock_put_work, 148 SMC_CLOSE_SOCK_PUT_DELAY); 149 } 150 release_sock(sk); 151 152 sock_put(sk); 153 out: 154 return rc; 155 } 156 157 static void smc_destruct(struct sock *sk) 158 { 159 if (sk->sk_state != SMC_CLOSED) 160 return; 161 if (!sock_flag(sk, SOCK_DEAD)) 162 return; 163 164 sk_refcnt_debug_dec(sk); 165 } 166 167 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock) 168 { 169 struct smc_sock *smc; 170 struct sock *sk; 171 172 sk = sk_alloc(net, PF_SMC, GFP_KERNEL, &smc_proto, 0); 173 if (!sk) 174 return NULL; 175 176 sock_init_data(sock, sk); /* sets sk_refcnt to 1 */ 177 sk->sk_state = SMC_INIT; 178 sk->sk_destruct = smc_destruct; 179 sk->sk_protocol = SMCPROTO_SMC; 180 smc = smc_sk(sk); 181 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work); 182 INIT_LIST_HEAD(&smc->accept_q); 183 spin_lock_init(&smc->accept_q_lock); 184 INIT_DELAYED_WORK(&smc->sock_put_work, smc_close_sock_put_work); 185 sk->sk_prot->hash(sk); 186 sk_refcnt_debug_inc(sk); 187 188 return sk; 189 } 190 191 static int smc_bind(struct socket *sock, struct sockaddr *uaddr, 192 int addr_len) 193 { 194 struct sockaddr_in *addr = (struct sockaddr_in *)uaddr; 195 struct sock *sk = sock->sk; 196 struct smc_sock *smc; 197 int rc; 198 199 smc = smc_sk(sk); 200 201 /* replicate tests from inet_bind(), to be safe wrt. future changes */ 202 rc = -EINVAL; 203 if (addr_len < sizeof(struct sockaddr_in)) 204 goto out; 205 206 rc = -EAFNOSUPPORT; 207 /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */ 208 if ((addr->sin_family != AF_INET) && 209 ((addr->sin_family != AF_UNSPEC) || 210 (addr->sin_addr.s_addr != htonl(INADDR_ANY)))) 211 goto out; 212 213 lock_sock(sk); 214 215 /* Check if socket is already active */ 216 rc = -EINVAL; 217 if (sk->sk_state != SMC_INIT) 218 goto out_rel; 219 220 smc->clcsock->sk->sk_reuse = sk->sk_reuse; 221 rc = kernel_bind(smc->clcsock, uaddr, addr_len); 222 223 out_rel: 224 release_sock(sk); 225 out: 226 return rc; 227 } 228 229 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk, 230 unsigned long mask) 231 { 232 /* options we don't get control via setsockopt for */ 233 nsk->sk_type = osk->sk_type; 234 nsk->sk_sndbuf = osk->sk_sndbuf; 235 nsk->sk_rcvbuf = osk->sk_rcvbuf; 236 nsk->sk_sndtimeo = osk->sk_sndtimeo; 237 nsk->sk_rcvtimeo = osk->sk_rcvtimeo; 238 nsk->sk_mark = osk->sk_mark; 239 nsk->sk_priority = osk->sk_priority; 240 nsk->sk_rcvlowat = osk->sk_rcvlowat; 241 nsk->sk_bound_dev_if = osk->sk_bound_dev_if; 242 nsk->sk_err = osk->sk_err; 243 244 nsk->sk_flags &= ~mask; 245 nsk->sk_flags |= osk->sk_flags & mask; 246 } 247 248 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \ 249 (1UL << SOCK_KEEPOPEN) | \ 250 (1UL << SOCK_LINGER) | \ 251 (1UL << SOCK_BROADCAST) | \ 252 (1UL << SOCK_TIMESTAMP) | \ 253 (1UL << SOCK_DBG) | \ 254 (1UL << SOCK_RCVTSTAMP) | \ 255 (1UL << SOCK_RCVTSTAMPNS) | \ 256 (1UL << SOCK_LOCALROUTE) | \ 257 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \ 258 (1UL << SOCK_RXQ_OVFL) | \ 259 (1UL << SOCK_WIFI_STATUS) | \ 260 (1UL << SOCK_NOFCS) | \ 261 (1UL << SOCK_FILTER_LOCKED)) 262 /* copy only relevant settings and flags of SOL_SOCKET level from smc to 263 * clc socket (since smc is not called for these options from net/core) 264 */ 265 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc) 266 { 267 smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC); 268 } 269 270 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \ 271 (1UL << SOCK_KEEPOPEN) | \ 272 (1UL << SOCK_LINGER) | \ 273 (1UL << SOCK_DBG)) 274 /* copy only settings and flags relevant for smc from clc to smc socket */ 275 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc) 276 { 277 smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC); 278 } 279 280 /* determine subnet and mask of internal TCP socket */ 281 int smc_netinfo_by_tcpsk(struct socket *clcsock, 282 __be32 *subnet, u8 *prefix_len) 283 { 284 struct dst_entry *dst = sk_dst_get(clcsock->sk); 285 struct in_device *in_dev; 286 struct sockaddr_in addr; 287 int rc = -ENOENT; 288 int len; 289 290 if (!dst) { 291 rc = -ENOTCONN; 292 goto out; 293 } 294 if (!dst->dev) { 295 rc = -ENODEV; 296 goto out_rel; 297 } 298 299 /* get address to which the internal TCP socket is bound */ 300 kernel_getsockname(clcsock, (struct sockaddr *)&addr, &len); 301 /* analyze IPv4 specific data of net_device belonging to TCP socket */ 302 rcu_read_lock(); 303 in_dev = __in_dev_get_rcu(dst->dev); 304 for_ifa(in_dev) { 305 if (!inet_ifa_match(addr.sin_addr.s_addr, ifa)) 306 continue; 307 *prefix_len = inet_mask_len(ifa->ifa_mask); 308 *subnet = ifa->ifa_address & ifa->ifa_mask; 309 rc = 0; 310 break; 311 } endfor_ifa(in_dev); 312 rcu_read_unlock(); 313 314 out_rel: 315 dst_release(dst); 316 out: 317 return rc; 318 } 319 320 static int smc_clnt_conf_first_link(struct smc_sock *smc, union ib_gid *gid) 321 { 322 struct smc_link_group *lgr = smc->conn.lgr; 323 struct smc_link *link; 324 int rest; 325 int rc; 326 327 link = &lgr->lnk[SMC_SINGLE_LINK]; 328 /* receive CONFIRM LINK request from server over RoCE fabric */ 329 rest = wait_for_completion_interruptible_timeout( 330 &link->llc_confirm, 331 SMC_LLC_WAIT_FIRST_TIME); 332 if (rest <= 0) { 333 struct smc_clc_msg_decline dclc; 334 335 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), 336 SMC_CLC_DECLINE); 337 return rc; 338 } 339 340 rc = smc_ib_modify_qp_rts(link); 341 if (rc) 342 return SMC_CLC_DECL_INTERR; 343 344 smc_wr_remember_qp_attr(link); 345 346 rc = smc_wr_reg_send(link, 347 smc->conn.rmb_desc->mr_rx[SMC_SINGLE_LINK]); 348 if (rc) 349 return SMC_CLC_DECL_INTERR; 350 351 /* send CONFIRM LINK response over RoCE fabric */ 352 rc = smc_llc_send_confirm_link(link, 353 link->smcibdev->mac[link->ibport - 1], 354 gid, SMC_LLC_RESP); 355 if (rc < 0) 356 return SMC_CLC_DECL_TCL; 357 358 return rc; 359 } 360 361 static void smc_conn_save_peer_info(struct smc_sock *smc, 362 struct smc_clc_msg_accept_confirm *clc) 363 { 364 smc->conn.peer_conn_idx = clc->conn_idx; 365 smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token); 366 smc->conn.peer_rmbe_size = smc_uncompress_bufsize(clc->rmbe_size); 367 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size); 368 } 369 370 static void smc_link_save_peer_info(struct smc_link *link, 371 struct smc_clc_msg_accept_confirm *clc) 372 { 373 link->peer_qpn = ntoh24(clc->qpn); 374 memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE); 375 memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac)); 376 link->peer_psn = ntoh24(clc->psn); 377 link->peer_mtu = clc->qp_mtu; 378 } 379 380 /* setup for RDMA connection of client */ 381 static int smc_connect_rdma(struct smc_sock *smc) 382 { 383 struct sockaddr_in *inaddr = (struct sockaddr_in *)smc->addr; 384 struct smc_clc_msg_accept_confirm aclc; 385 int local_contact = SMC_FIRST_CONTACT; 386 struct smc_ib_device *smcibdev; 387 struct smc_link *link; 388 u8 srv_first_contact; 389 int reason_code = 0; 390 int rc = 0; 391 u8 ibport; 392 393 if (!tcp_sk(smc->clcsock->sk)->syn_smc) { 394 /* peer has not signalled SMC-capability */ 395 smc->use_fallback = true; 396 goto out_connected; 397 } 398 399 /* IPSec connections opt out of SMC-R optimizations */ 400 if (using_ipsec(smc)) { 401 reason_code = SMC_CLC_DECL_IPSEC; 402 goto decline_rdma; 403 } 404 405 /* PNET table look up: search active ib_device and port 406 * within same PNETID that also contains the ethernet device 407 * used for the internal TCP socket 408 */ 409 smc_pnet_find_roce_resource(smc->clcsock->sk, &smcibdev, &ibport); 410 if (!smcibdev) { 411 reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */ 412 goto decline_rdma; 413 } 414 415 /* do inband token exchange */ 416 reason_code = smc_clc_send_proposal(smc, smcibdev, ibport); 417 if (reason_code < 0) { 418 rc = reason_code; 419 goto out_err; 420 } 421 if (reason_code > 0) /* configuration error */ 422 goto decline_rdma; 423 /* receive SMC Accept CLC message */ 424 reason_code = smc_clc_wait_msg(smc, &aclc, sizeof(aclc), 425 SMC_CLC_ACCEPT); 426 if (reason_code < 0) { 427 rc = reason_code; 428 goto out_err; 429 } 430 if (reason_code > 0) 431 goto decline_rdma; 432 433 srv_first_contact = aclc.hdr.flag; 434 mutex_lock(&smc_create_lgr_pending); 435 local_contact = smc_conn_create(smc, inaddr->sin_addr.s_addr, smcibdev, 436 ibport, &aclc.lcl, srv_first_contact); 437 if (local_contact < 0) { 438 rc = local_contact; 439 if (rc == -ENOMEM) 440 reason_code = SMC_CLC_DECL_MEM;/* insufficient memory*/ 441 else if (rc == -ENOLINK) 442 reason_code = SMC_CLC_DECL_SYNCERR; /* synchr. error */ 443 goto decline_rdma_unlock; 444 } 445 link = &smc->conn.lgr->lnk[SMC_SINGLE_LINK]; 446 447 smc_conn_save_peer_info(smc, &aclc); 448 449 /* create send buffer and rmb */ 450 rc = smc_buf_create(smc); 451 if (rc) { 452 reason_code = SMC_CLC_DECL_MEM; 453 goto decline_rdma_unlock; 454 } 455 456 if (local_contact == SMC_FIRST_CONTACT) 457 smc_link_save_peer_info(link, &aclc); 458 459 rc = smc_rmb_rtoken_handling(&smc->conn, &aclc); 460 if (rc) { 461 reason_code = SMC_CLC_DECL_INTERR; 462 goto decline_rdma_unlock; 463 } 464 465 smc_close_init(smc); 466 smc_rx_init(smc); 467 468 if (local_contact == SMC_FIRST_CONTACT) { 469 rc = smc_ib_ready_link(link); 470 if (rc) { 471 reason_code = SMC_CLC_DECL_INTERR; 472 goto decline_rdma_unlock; 473 } 474 } else { 475 struct smc_buf_desc *buf_desc = smc->conn.rmb_desc; 476 477 if (!buf_desc->reused) { 478 /* register memory region for new rmb */ 479 rc = smc_wr_reg_send(link, 480 buf_desc->mr_rx[SMC_SINGLE_LINK]); 481 if (rc) { 482 reason_code = SMC_CLC_DECL_INTERR; 483 goto decline_rdma_unlock; 484 } 485 } 486 } 487 smc_rmb_sync_sg_for_device(&smc->conn); 488 489 rc = smc_clc_send_confirm(smc); 490 if (rc) 491 goto out_err_unlock; 492 493 if (local_contact == SMC_FIRST_CONTACT) { 494 /* QP confirmation over RoCE fabric */ 495 reason_code = smc_clnt_conf_first_link( 496 smc, &smcibdev->gid[ibport - 1]); 497 if (reason_code < 0) { 498 rc = reason_code; 499 goto out_err_unlock; 500 } 501 if (reason_code > 0) 502 goto decline_rdma_unlock; 503 } 504 505 mutex_unlock(&smc_create_lgr_pending); 506 smc_tx_init(smc); 507 508 out_connected: 509 smc_copy_sock_settings_to_clc(smc); 510 if (smc->sk.sk_state == SMC_INIT) 511 smc->sk.sk_state = SMC_ACTIVE; 512 513 return rc ? rc : local_contact; 514 515 decline_rdma_unlock: 516 mutex_unlock(&smc_create_lgr_pending); 517 smc_conn_free(&smc->conn); 518 decline_rdma: 519 /* RDMA setup failed, switch back to TCP */ 520 smc->use_fallback = true; 521 if (reason_code && (reason_code != SMC_CLC_DECL_REPLY)) { 522 rc = smc_clc_send_decline(smc, reason_code); 523 if (rc < 0) 524 goto out_err; 525 } 526 goto out_connected; 527 528 out_err_unlock: 529 mutex_unlock(&smc_create_lgr_pending); 530 smc_conn_free(&smc->conn); 531 out_err: 532 return rc; 533 } 534 535 static int smc_connect(struct socket *sock, struct sockaddr *addr, 536 int alen, int flags) 537 { 538 struct sock *sk = sock->sk; 539 struct smc_sock *smc; 540 int rc = -EINVAL; 541 542 smc = smc_sk(sk); 543 544 /* separate smc parameter checking to be safe */ 545 if (alen < sizeof(addr->sa_family)) 546 goto out_err; 547 if (addr->sa_family != AF_INET) 548 goto out_err; 549 smc->addr = addr; /* needed for nonblocking connect */ 550 551 lock_sock(sk); 552 switch (sk->sk_state) { 553 default: 554 goto out; 555 case SMC_ACTIVE: 556 rc = -EISCONN; 557 goto out; 558 case SMC_INIT: 559 rc = 0; 560 break; 561 } 562 563 smc_copy_sock_settings_to_clc(smc); 564 tcp_sk(smc->clcsock->sk)->syn_smc = 1; 565 rc = kernel_connect(smc->clcsock, addr, alen, flags); 566 if (rc) 567 goto out; 568 569 /* setup RDMA connection */ 570 rc = smc_connect_rdma(smc); 571 if (rc < 0) 572 goto out; 573 else 574 rc = 0; /* success cases including fallback */ 575 576 out: 577 release_sock(sk); 578 out_err: 579 return rc; 580 } 581 582 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc) 583 { 584 struct sock *sk = &lsmc->sk; 585 struct socket *new_clcsock; 586 struct sock *new_sk; 587 int rc; 588 589 release_sock(&lsmc->sk); 590 new_sk = smc_sock_alloc(sock_net(sk), NULL); 591 if (!new_sk) { 592 rc = -ENOMEM; 593 lsmc->sk.sk_err = ENOMEM; 594 *new_smc = NULL; 595 lock_sock(&lsmc->sk); 596 goto out; 597 } 598 *new_smc = smc_sk(new_sk); 599 600 rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0); 601 lock_sock(&lsmc->sk); 602 if (rc < 0) { 603 lsmc->sk.sk_err = -rc; 604 new_sk->sk_state = SMC_CLOSED; 605 sock_set_flag(new_sk, SOCK_DEAD); 606 sk->sk_prot->unhash(new_sk); 607 sock_put(new_sk); 608 *new_smc = NULL; 609 goto out; 610 } 611 if (lsmc->sk.sk_state == SMC_CLOSED) { 612 if (new_clcsock) 613 sock_release(new_clcsock); 614 new_sk->sk_state = SMC_CLOSED; 615 sock_set_flag(new_sk, SOCK_DEAD); 616 sk->sk_prot->unhash(new_sk); 617 sock_put(new_sk); 618 *new_smc = NULL; 619 goto out; 620 } 621 622 (*new_smc)->clcsock = new_clcsock; 623 out: 624 return rc; 625 } 626 627 /* add a just created sock to the accept queue of the listen sock as 628 * candidate for a following socket accept call from user space 629 */ 630 static void smc_accept_enqueue(struct sock *parent, struct sock *sk) 631 { 632 struct smc_sock *par = smc_sk(parent); 633 634 sock_hold(sk); 635 spin_lock(&par->accept_q_lock); 636 list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q); 637 spin_unlock(&par->accept_q_lock); 638 sk_acceptq_added(parent); 639 } 640 641 /* remove a socket from the accept queue of its parental listening socket */ 642 static void smc_accept_unlink(struct sock *sk) 643 { 644 struct smc_sock *par = smc_sk(sk)->listen_smc; 645 646 spin_lock(&par->accept_q_lock); 647 list_del_init(&smc_sk(sk)->accept_q); 648 spin_unlock(&par->accept_q_lock); 649 sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk); 650 sock_put(sk); 651 } 652 653 /* remove a sock from the accept queue to bind it to a new socket created 654 * for a socket accept call from user space 655 */ 656 struct sock *smc_accept_dequeue(struct sock *parent, 657 struct socket *new_sock) 658 { 659 struct smc_sock *isk, *n; 660 struct sock *new_sk; 661 662 list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) { 663 new_sk = (struct sock *)isk; 664 665 smc_accept_unlink(new_sk); 666 if (new_sk->sk_state == SMC_CLOSED) { 667 new_sk->sk_prot->unhash(new_sk); 668 sock_put(new_sk); 669 continue; 670 } 671 if (new_sock) 672 sock_graft(new_sk, new_sock); 673 return new_sk; 674 } 675 return NULL; 676 } 677 678 /* clean up for a created but never accepted sock */ 679 void smc_close_non_accepted(struct sock *sk) 680 { 681 struct smc_sock *smc = smc_sk(sk); 682 683 sock_hold(sk); 684 lock_sock(sk); 685 if (!sk->sk_lingertime) 686 /* wait for peer closing */ 687 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT; 688 if (smc->use_fallback) { 689 sk->sk_state = SMC_CLOSED; 690 } else { 691 smc_close_active(smc); 692 sock_set_flag(sk, SOCK_DEAD); 693 sk->sk_shutdown |= SHUTDOWN_MASK; 694 } 695 if (smc->clcsock) { 696 struct socket *tcp; 697 698 tcp = smc->clcsock; 699 smc->clcsock = NULL; 700 sock_release(tcp); 701 } 702 if (smc->use_fallback) { 703 schedule_delayed_work(&smc->sock_put_work, TCP_TIMEWAIT_LEN); 704 } else if (sk->sk_state == SMC_CLOSED) { 705 smc_conn_free(&smc->conn); 706 schedule_delayed_work(&smc->sock_put_work, 707 SMC_CLOSE_SOCK_PUT_DELAY); 708 } 709 release_sock(sk); 710 sock_put(sk); 711 } 712 713 static int smc_serv_conf_first_link(struct smc_sock *smc) 714 { 715 struct smc_link_group *lgr = smc->conn.lgr; 716 struct smc_link *link; 717 int rest; 718 int rc; 719 720 link = &lgr->lnk[SMC_SINGLE_LINK]; 721 722 rc = smc_wr_reg_send(link, 723 smc->conn.rmb_desc->mr_rx[SMC_SINGLE_LINK]); 724 if (rc) 725 return SMC_CLC_DECL_INTERR; 726 727 /* send CONFIRM LINK request to client over the RoCE fabric */ 728 rc = smc_llc_send_confirm_link(link, 729 link->smcibdev->mac[link->ibport - 1], 730 &link->smcibdev->gid[link->ibport - 1], 731 SMC_LLC_REQ); 732 if (rc < 0) 733 return SMC_CLC_DECL_TCL; 734 735 /* receive CONFIRM LINK response from client over the RoCE fabric */ 736 rest = wait_for_completion_interruptible_timeout( 737 &link->llc_confirm_resp, 738 SMC_LLC_WAIT_FIRST_TIME); 739 if (rest <= 0) { 740 struct smc_clc_msg_decline dclc; 741 742 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), 743 SMC_CLC_DECLINE); 744 } 745 746 return rc; 747 } 748 749 /* setup for RDMA connection of server */ 750 static void smc_listen_work(struct work_struct *work) 751 { 752 struct smc_sock *new_smc = container_of(work, struct smc_sock, 753 smc_listen_work); 754 struct smc_clc_msg_proposal_prefix *pclc_prfx; 755 struct socket *newclcsock = new_smc->clcsock; 756 struct smc_sock *lsmc = new_smc->listen_smc; 757 struct smc_clc_msg_accept_confirm cclc; 758 int local_contact = SMC_REUSE_CONTACT; 759 struct sock *newsmcsk = &new_smc->sk; 760 struct smc_clc_msg_proposal *pclc; 761 struct smc_ib_device *smcibdev; 762 struct sockaddr_in peeraddr; 763 u8 buf[SMC_CLC_MAX_LEN]; 764 struct smc_link *link; 765 int reason_code = 0; 766 int rc = 0, len; 767 __be32 subnet; 768 u8 prefix_len; 769 u8 ibport; 770 771 /* check if peer is smc capable */ 772 if (!tcp_sk(newclcsock->sk)->syn_smc) { 773 new_smc->use_fallback = true; 774 goto out_connected; 775 } 776 777 /* do inband token exchange - 778 *wait for and receive SMC Proposal CLC message 779 */ 780 reason_code = smc_clc_wait_msg(new_smc, &buf, sizeof(buf), 781 SMC_CLC_PROPOSAL); 782 if (reason_code < 0) 783 goto out_err; 784 if (reason_code > 0) 785 goto decline_rdma; 786 787 /* IPSec connections opt out of SMC-R optimizations */ 788 if (using_ipsec(new_smc)) { 789 reason_code = SMC_CLC_DECL_IPSEC; 790 goto decline_rdma; 791 } 792 793 /* PNET table look up: search active ib_device and port 794 * within same PNETID that also contains the ethernet device 795 * used for the internal TCP socket 796 */ 797 smc_pnet_find_roce_resource(newclcsock->sk, &smcibdev, &ibport); 798 if (!smcibdev) { 799 reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */ 800 goto decline_rdma; 801 } 802 803 /* determine subnet and mask from internal TCP socket */ 804 rc = smc_netinfo_by_tcpsk(newclcsock, &subnet, &prefix_len); 805 if (rc) { 806 reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */ 807 goto decline_rdma; 808 } 809 810 pclc = (struct smc_clc_msg_proposal *)&buf; 811 pclc_prfx = smc_clc_proposal_get_prefix(pclc); 812 if (pclc_prfx->outgoing_subnet != subnet || 813 pclc_prfx->prefix_len != prefix_len) { 814 reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */ 815 goto decline_rdma; 816 } 817 818 /* get address of the peer connected to the internal TCP socket */ 819 kernel_getpeername(newclcsock, (struct sockaddr *)&peeraddr, &len); 820 821 /* allocate connection / link group */ 822 mutex_lock(&smc_create_lgr_pending); 823 local_contact = smc_conn_create(new_smc, peeraddr.sin_addr.s_addr, 824 smcibdev, ibport, &pclc->lcl, 0); 825 if (local_contact < 0) { 826 rc = local_contact; 827 if (rc == -ENOMEM) 828 reason_code = SMC_CLC_DECL_MEM;/* insufficient memory*/ 829 goto decline_rdma_unlock; 830 } 831 link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK]; 832 833 /* create send buffer and rmb */ 834 rc = smc_buf_create(new_smc); 835 if (rc) { 836 reason_code = SMC_CLC_DECL_MEM; 837 goto decline_rdma_unlock; 838 } 839 840 smc_close_init(new_smc); 841 smc_rx_init(new_smc); 842 843 if (local_contact != SMC_FIRST_CONTACT) { 844 struct smc_buf_desc *buf_desc = new_smc->conn.rmb_desc; 845 846 if (!buf_desc->reused) { 847 /* register memory region for new rmb */ 848 rc = smc_wr_reg_send(link, 849 buf_desc->mr_rx[SMC_SINGLE_LINK]); 850 if (rc) { 851 reason_code = SMC_CLC_DECL_INTERR; 852 goto decline_rdma_unlock; 853 } 854 } 855 } 856 smc_rmb_sync_sg_for_device(&new_smc->conn); 857 858 rc = smc_clc_send_accept(new_smc, local_contact); 859 if (rc) 860 goto out_err_unlock; 861 862 /* receive SMC Confirm CLC message */ 863 reason_code = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc), 864 SMC_CLC_CONFIRM); 865 if (reason_code < 0) 866 goto out_err_unlock; 867 if (reason_code > 0) 868 goto decline_rdma_unlock; 869 smc_conn_save_peer_info(new_smc, &cclc); 870 if (local_contact == SMC_FIRST_CONTACT) 871 smc_link_save_peer_info(link, &cclc); 872 873 rc = smc_rmb_rtoken_handling(&new_smc->conn, &cclc); 874 if (rc) { 875 reason_code = SMC_CLC_DECL_INTERR; 876 goto decline_rdma_unlock; 877 } 878 879 if (local_contact == SMC_FIRST_CONTACT) { 880 rc = smc_ib_ready_link(link); 881 if (rc) { 882 reason_code = SMC_CLC_DECL_INTERR; 883 goto decline_rdma_unlock; 884 } 885 /* QP confirmation over RoCE fabric */ 886 reason_code = smc_serv_conf_first_link(new_smc); 887 if (reason_code < 0) 888 /* peer is not aware of a problem */ 889 goto out_err_unlock; 890 if (reason_code > 0) 891 goto decline_rdma_unlock; 892 } 893 894 smc_tx_init(new_smc); 895 mutex_unlock(&smc_create_lgr_pending); 896 897 out_connected: 898 sk_refcnt_debug_inc(newsmcsk); 899 if (newsmcsk->sk_state == SMC_INIT) 900 newsmcsk->sk_state = SMC_ACTIVE; 901 enqueue: 902 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING); 903 if (lsmc->sk.sk_state == SMC_LISTEN) { 904 smc_accept_enqueue(&lsmc->sk, newsmcsk); 905 } else { /* no longer listening */ 906 smc_close_non_accepted(newsmcsk); 907 } 908 release_sock(&lsmc->sk); 909 910 /* Wake up accept */ 911 lsmc->sk.sk_data_ready(&lsmc->sk); 912 sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */ 913 return; 914 915 decline_rdma_unlock: 916 mutex_unlock(&smc_create_lgr_pending); 917 decline_rdma: 918 /* RDMA setup failed, switch back to TCP */ 919 smc_conn_free(&new_smc->conn); 920 new_smc->use_fallback = true; 921 if (reason_code && (reason_code != SMC_CLC_DECL_REPLY)) { 922 if (smc_clc_send_decline(new_smc, reason_code) < 0) 923 goto out_err; 924 } 925 goto out_connected; 926 927 out_err_unlock: 928 mutex_unlock(&smc_create_lgr_pending); 929 out_err: 930 newsmcsk->sk_state = SMC_CLOSED; 931 smc_conn_free(&new_smc->conn); 932 goto enqueue; /* queue new sock with sk_err set */ 933 } 934 935 static void smc_tcp_listen_work(struct work_struct *work) 936 { 937 struct smc_sock *lsmc = container_of(work, struct smc_sock, 938 tcp_listen_work); 939 struct smc_sock *new_smc; 940 int rc = 0; 941 942 lock_sock(&lsmc->sk); 943 while (lsmc->sk.sk_state == SMC_LISTEN) { 944 rc = smc_clcsock_accept(lsmc, &new_smc); 945 if (rc) 946 goto out; 947 if (!new_smc) 948 continue; 949 950 new_smc->listen_smc = lsmc; 951 new_smc->use_fallback = false; /* assume rdma capability first*/ 952 sock_hold(&lsmc->sk); /* sock_put in smc_listen_work */ 953 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work); 954 smc_copy_sock_settings_to_smc(new_smc); 955 schedule_work(&new_smc->smc_listen_work); 956 } 957 958 out: 959 release_sock(&lsmc->sk); 960 lsmc->sk.sk_data_ready(&lsmc->sk); /* no more listening, wake accept */ 961 } 962 963 static int smc_listen(struct socket *sock, int backlog) 964 { 965 struct sock *sk = sock->sk; 966 struct smc_sock *smc; 967 int rc; 968 969 smc = smc_sk(sk); 970 lock_sock(sk); 971 972 rc = -EINVAL; 973 if ((sk->sk_state != SMC_INIT) && (sk->sk_state != SMC_LISTEN)) 974 goto out; 975 976 rc = 0; 977 if (sk->sk_state == SMC_LISTEN) { 978 sk->sk_max_ack_backlog = backlog; 979 goto out; 980 } 981 /* some socket options are handled in core, so we could not apply 982 * them to the clc socket -- copy smc socket options to clc socket 983 */ 984 smc_copy_sock_settings_to_clc(smc); 985 tcp_sk(smc->clcsock->sk)->syn_smc = 1; 986 987 rc = kernel_listen(smc->clcsock, backlog); 988 if (rc) 989 goto out; 990 sk->sk_max_ack_backlog = backlog; 991 sk->sk_ack_backlog = 0; 992 sk->sk_state = SMC_LISTEN; 993 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work); 994 schedule_work(&smc->tcp_listen_work); 995 996 out: 997 release_sock(sk); 998 return rc; 999 } 1000 1001 static int smc_accept(struct socket *sock, struct socket *new_sock, 1002 int flags, bool kern) 1003 { 1004 struct sock *sk = sock->sk, *nsk; 1005 DECLARE_WAITQUEUE(wait, current); 1006 struct smc_sock *lsmc; 1007 long timeo; 1008 int rc = 0; 1009 1010 lsmc = smc_sk(sk); 1011 lock_sock(sk); 1012 1013 if (lsmc->sk.sk_state != SMC_LISTEN) { 1014 rc = -EINVAL; 1015 goto out; 1016 } 1017 1018 /* Wait for an incoming connection */ 1019 timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); 1020 add_wait_queue_exclusive(sk_sleep(sk), &wait); 1021 while (!(nsk = smc_accept_dequeue(sk, new_sock))) { 1022 set_current_state(TASK_INTERRUPTIBLE); 1023 if (!timeo) { 1024 rc = -EAGAIN; 1025 break; 1026 } 1027 release_sock(sk); 1028 timeo = schedule_timeout(timeo); 1029 /* wakeup by sk_data_ready in smc_listen_work() */ 1030 sched_annotate_sleep(); 1031 lock_sock(sk); 1032 if (signal_pending(current)) { 1033 rc = sock_intr_errno(timeo); 1034 break; 1035 } 1036 } 1037 set_current_state(TASK_RUNNING); 1038 remove_wait_queue(sk_sleep(sk), &wait); 1039 1040 if (!rc) 1041 rc = sock_error(nsk); 1042 1043 out: 1044 release_sock(sk); 1045 return rc; 1046 } 1047 1048 static int smc_getname(struct socket *sock, struct sockaddr *addr, 1049 int *len, int peer) 1050 { 1051 struct smc_sock *smc; 1052 1053 if (peer && (sock->sk->sk_state != SMC_ACTIVE) && 1054 (sock->sk->sk_state != SMC_APPCLOSEWAIT1)) 1055 return -ENOTCONN; 1056 1057 smc = smc_sk(sock->sk); 1058 1059 return smc->clcsock->ops->getname(smc->clcsock, addr, len, peer); 1060 } 1061 1062 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) 1063 { 1064 struct sock *sk = sock->sk; 1065 struct smc_sock *smc; 1066 int rc = -EPIPE; 1067 1068 smc = smc_sk(sk); 1069 lock_sock(sk); 1070 if ((sk->sk_state != SMC_ACTIVE) && 1071 (sk->sk_state != SMC_APPCLOSEWAIT1) && 1072 (sk->sk_state != SMC_INIT)) 1073 goto out; 1074 if (smc->use_fallback) 1075 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len); 1076 else 1077 rc = smc_tx_sendmsg(smc, msg, len); 1078 out: 1079 release_sock(sk); 1080 return rc; 1081 } 1082 1083 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, 1084 int flags) 1085 { 1086 struct sock *sk = sock->sk; 1087 struct smc_sock *smc; 1088 int rc = -ENOTCONN; 1089 1090 smc = smc_sk(sk); 1091 lock_sock(sk); 1092 if ((sk->sk_state == SMC_INIT) || 1093 (sk->sk_state == SMC_LISTEN) || 1094 (sk->sk_state == SMC_CLOSED)) 1095 goto out; 1096 1097 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) { 1098 rc = 0; 1099 goto out; 1100 } 1101 1102 if (smc->use_fallback) 1103 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags); 1104 else 1105 rc = smc_rx_recvmsg(smc, msg, len, flags); 1106 1107 out: 1108 release_sock(sk); 1109 return rc; 1110 } 1111 1112 static unsigned int smc_accept_poll(struct sock *parent) 1113 { 1114 struct smc_sock *isk; 1115 struct sock *sk; 1116 1117 lock_sock(parent); 1118 list_for_each_entry(isk, &smc_sk(parent)->accept_q, accept_q) { 1119 sk = (struct sock *)isk; 1120 1121 if (sk->sk_state == SMC_ACTIVE) { 1122 release_sock(parent); 1123 return POLLIN | POLLRDNORM; 1124 } 1125 } 1126 release_sock(parent); 1127 1128 return 0; 1129 } 1130 1131 static unsigned int smc_poll(struct file *file, struct socket *sock, 1132 poll_table *wait) 1133 { 1134 struct sock *sk = sock->sk; 1135 unsigned int mask = 0; 1136 struct smc_sock *smc; 1137 int rc; 1138 1139 smc = smc_sk(sock->sk); 1140 if ((sk->sk_state == SMC_INIT) || smc->use_fallback) { 1141 /* delegate to CLC child sock */ 1142 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait); 1143 /* if non-blocking connect finished ... */ 1144 lock_sock(sk); 1145 if ((sk->sk_state == SMC_INIT) && (mask & POLLOUT)) { 1146 sk->sk_err = smc->clcsock->sk->sk_err; 1147 if (sk->sk_err) { 1148 mask |= POLLERR; 1149 } else { 1150 rc = smc_connect_rdma(smc); 1151 if (rc < 0) 1152 mask |= POLLERR; 1153 else 1154 /* success cases including fallback */ 1155 mask |= POLLOUT | POLLWRNORM; 1156 } 1157 } 1158 release_sock(sk); 1159 } else { 1160 sock_poll_wait(file, sk_sleep(sk), wait); 1161 if (sk->sk_state == SMC_LISTEN) 1162 /* woken up by sk_data_ready in smc_listen_work() */ 1163 mask |= smc_accept_poll(sk); 1164 if (sk->sk_err) 1165 mask |= POLLERR; 1166 if (atomic_read(&smc->conn.sndbuf_space) || 1167 (sk->sk_shutdown & SEND_SHUTDOWN)) { 1168 mask |= POLLOUT | POLLWRNORM; 1169 } else { 1170 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 1171 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1172 } 1173 if (atomic_read(&smc->conn.bytes_to_rcv)) 1174 mask |= POLLIN | POLLRDNORM; 1175 if ((sk->sk_shutdown == SHUTDOWN_MASK) || 1176 (sk->sk_state == SMC_CLOSED)) 1177 mask |= POLLHUP; 1178 if (sk->sk_shutdown & RCV_SHUTDOWN) 1179 mask |= POLLIN | POLLRDNORM | POLLRDHUP; 1180 if (sk->sk_state == SMC_APPCLOSEWAIT1) 1181 mask |= POLLIN; 1182 1183 } 1184 1185 return mask; 1186 } 1187 1188 static int smc_shutdown(struct socket *sock, int how) 1189 { 1190 struct sock *sk = sock->sk; 1191 struct smc_sock *smc; 1192 int rc = -EINVAL; 1193 int rc1 = 0; 1194 1195 smc = smc_sk(sk); 1196 1197 if ((how < SHUT_RD) || (how > SHUT_RDWR)) 1198 return rc; 1199 1200 lock_sock(sk); 1201 1202 rc = -ENOTCONN; 1203 if ((sk->sk_state != SMC_LISTEN) && 1204 (sk->sk_state != SMC_ACTIVE) && 1205 (sk->sk_state != SMC_PEERCLOSEWAIT1) && 1206 (sk->sk_state != SMC_PEERCLOSEWAIT2) && 1207 (sk->sk_state != SMC_APPCLOSEWAIT1) && 1208 (sk->sk_state != SMC_APPCLOSEWAIT2) && 1209 (sk->sk_state != SMC_APPFINCLOSEWAIT)) 1210 goto out; 1211 if (smc->use_fallback) { 1212 rc = kernel_sock_shutdown(smc->clcsock, how); 1213 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown; 1214 if (sk->sk_shutdown == SHUTDOWN_MASK) 1215 sk->sk_state = SMC_CLOSED; 1216 goto out; 1217 } 1218 switch (how) { 1219 case SHUT_RDWR: /* shutdown in both directions */ 1220 rc = smc_close_active(smc); 1221 break; 1222 case SHUT_WR: 1223 rc = smc_close_shutdown_write(smc); 1224 break; 1225 case SHUT_RD: 1226 if (sk->sk_state == SMC_LISTEN) 1227 rc = smc_close_active(smc); 1228 else 1229 rc = 0; 1230 /* nothing more to do because peer is not involved */ 1231 break; 1232 } 1233 rc1 = kernel_sock_shutdown(smc->clcsock, how); 1234 /* map sock_shutdown_cmd constants to sk_shutdown value range */ 1235 sk->sk_shutdown |= how + 1; 1236 1237 out: 1238 release_sock(sk); 1239 return rc ? rc : rc1; 1240 } 1241 1242 static int smc_setsockopt(struct socket *sock, int level, int optname, 1243 char __user *optval, unsigned int optlen) 1244 { 1245 struct sock *sk = sock->sk; 1246 struct smc_sock *smc; 1247 1248 smc = smc_sk(sk); 1249 1250 /* generic setsockopts reaching us here always apply to the 1251 * CLC socket 1252 */ 1253 return smc->clcsock->ops->setsockopt(smc->clcsock, level, optname, 1254 optval, optlen); 1255 } 1256 1257 static int smc_getsockopt(struct socket *sock, int level, int optname, 1258 char __user *optval, int __user *optlen) 1259 { 1260 struct smc_sock *smc; 1261 1262 smc = smc_sk(sock->sk); 1263 /* socket options apply to the CLC socket */ 1264 return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname, 1265 optval, optlen); 1266 } 1267 1268 static int smc_ioctl(struct socket *sock, unsigned int cmd, 1269 unsigned long arg) 1270 { 1271 struct smc_sock *smc; 1272 1273 smc = smc_sk(sock->sk); 1274 if (smc->use_fallback) 1275 return smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg); 1276 else 1277 return sock_no_ioctl(sock, cmd, arg); 1278 } 1279 1280 static ssize_t smc_sendpage(struct socket *sock, struct page *page, 1281 int offset, size_t size, int flags) 1282 { 1283 struct sock *sk = sock->sk; 1284 struct smc_sock *smc; 1285 int rc = -EPIPE; 1286 1287 smc = smc_sk(sk); 1288 lock_sock(sk); 1289 if (sk->sk_state != SMC_ACTIVE) 1290 goto out; 1291 if (smc->use_fallback) 1292 rc = kernel_sendpage(smc->clcsock, page, offset, 1293 size, flags); 1294 else 1295 rc = sock_no_sendpage(sock, page, offset, size, flags); 1296 1297 out: 1298 release_sock(sk); 1299 return rc; 1300 } 1301 1302 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos, 1303 struct pipe_inode_info *pipe, size_t len, 1304 unsigned int flags) 1305 { 1306 struct sock *sk = sock->sk; 1307 struct smc_sock *smc; 1308 int rc = -ENOTCONN; 1309 1310 smc = smc_sk(sk); 1311 lock_sock(sk); 1312 if ((sk->sk_state != SMC_ACTIVE) && (sk->sk_state != SMC_CLOSED)) 1313 goto out; 1314 if (smc->use_fallback) { 1315 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos, 1316 pipe, len, flags); 1317 } else { 1318 rc = -EOPNOTSUPP; 1319 } 1320 out: 1321 release_sock(sk); 1322 return rc; 1323 } 1324 1325 /* must look like tcp */ 1326 static const struct proto_ops smc_sock_ops = { 1327 .family = PF_SMC, 1328 .owner = THIS_MODULE, 1329 .release = smc_release, 1330 .bind = smc_bind, 1331 .connect = smc_connect, 1332 .socketpair = sock_no_socketpair, 1333 .accept = smc_accept, 1334 .getname = smc_getname, 1335 .poll = smc_poll, 1336 .ioctl = smc_ioctl, 1337 .listen = smc_listen, 1338 .shutdown = smc_shutdown, 1339 .setsockopt = smc_setsockopt, 1340 .getsockopt = smc_getsockopt, 1341 .sendmsg = smc_sendmsg, 1342 .recvmsg = smc_recvmsg, 1343 .mmap = sock_no_mmap, 1344 .sendpage = smc_sendpage, 1345 .splice_read = smc_splice_read, 1346 }; 1347 1348 static int smc_create(struct net *net, struct socket *sock, int protocol, 1349 int kern) 1350 { 1351 struct smc_sock *smc; 1352 struct sock *sk; 1353 int rc; 1354 1355 rc = -ESOCKTNOSUPPORT; 1356 if (sock->type != SOCK_STREAM) 1357 goto out; 1358 1359 rc = -EPROTONOSUPPORT; 1360 if ((protocol != IPPROTO_IP) && (protocol != IPPROTO_TCP)) 1361 goto out; 1362 1363 rc = -ENOBUFS; 1364 sock->ops = &smc_sock_ops; 1365 sk = smc_sock_alloc(net, sock); 1366 if (!sk) 1367 goto out; 1368 1369 /* create internal TCP socket for CLC handshake and fallback */ 1370 smc = smc_sk(sk); 1371 smc->use_fallback = false; /* assume rdma capability first */ 1372 rc = sock_create_kern(net, PF_INET, SOCK_STREAM, 1373 IPPROTO_TCP, &smc->clcsock); 1374 if (rc) 1375 sk_common_release(sk); 1376 smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE); 1377 smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE); 1378 1379 out: 1380 return rc; 1381 } 1382 1383 static const struct net_proto_family smc_sock_family_ops = { 1384 .family = PF_SMC, 1385 .owner = THIS_MODULE, 1386 .create = smc_create, 1387 }; 1388 1389 static int __init smc_init(void) 1390 { 1391 int rc; 1392 1393 rc = smc_pnet_init(); 1394 if (rc) 1395 return rc; 1396 1397 rc = smc_llc_init(); 1398 if (rc) { 1399 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc); 1400 goto out_pnet; 1401 } 1402 1403 rc = smc_cdc_init(); 1404 if (rc) { 1405 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc); 1406 goto out_pnet; 1407 } 1408 1409 rc = proto_register(&smc_proto, 1); 1410 if (rc) { 1411 pr_err("%s: proto_register fails with %d\n", __func__, rc); 1412 goto out_pnet; 1413 } 1414 1415 rc = sock_register(&smc_sock_family_ops); 1416 if (rc) { 1417 pr_err("%s: sock_register fails with %d\n", __func__, rc); 1418 goto out_proto; 1419 } 1420 INIT_HLIST_HEAD(&smc_v4_hashinfo.ht); 1421 1422 rc = smc_ib_register_client(); 1423 if (rc) { 1424 pr_err("%s: ib_register fails with %d\n", __func__, rc); 1425 goto out_sock; 1426 } 1427 1428 static_branch_enable(&tcp_have_smc); 1429 return 0; 1430 1431 out_sock: 1432 sock_unregister(PF_SMC); 1433 out_proto: 1434 proto_unregister(&smc_proto); 1435 out_pnet: 1436 smc_pnet_exit(); 1437 return rc; 1438 } 1439 1440 static void __exit smc_exit(void) 1441 { 1442 struct smc_link_group *lgr, *lg; 1443 LIST_HEAD(lgr_freeing_list); 1444 1445 spin_lock_bh(&smc_lgr_list.lock); 1446 if (!list_empty(&smc_lgr_list.list)) 1447 list_splice_init(&smc_lgr_list.list, &lgr_freeing_list); 1448 spin_unlock_bh(&smc_lgr_list.lock); 1449 list_for_each_entry_safe(lgr, lg, &lgr_freeing_list, list) { 1450 list_del_init(&lgr->list); 1451 smc_lgr_free(lgr); /* free link group */ 1452 } 1453 static_branch_disable(&tcp_have_smc); 1454 smc_ib_unregister_client(); 1455 sock_unregister(PF_SMC); 1456 proto_unregister(&smc_proto); 1457 smc_pnet_exit(); 1458 } 1459 1460 module_init(smc_init); 1461 module_exit(smc_exit); 1462 1463 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>"); 1464 MODULE_DESCRIPTION("smc socket address family"); 1465 MODULE_LICENSE("GPL"); 1466 MODULE_ALIAS_NETPROTO(PF_SMC); 1467