1 /* 2 * Shared Memory Communications over RDMA (SMC-R) and RoCE 3 * 4 * AF_SMC protocol family socket handler keeping the AF_INET sock address type 5 * applies to SOCK_STREAM sockets only 6 * offers an alternative communication option for TCP-protocol sockets 7 * applicable with RoCE-cards only 8 * 9 * Initial restrictions: 10 * - support for alternate links postponed 11 * 12 * Copyright IBM Corp. 2016, 2018 13 * 14 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> 15 * based on prototype from Frank Blaschka 16 */ 17 18 #define KMSG_COMPONENT "smc" 19 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 20 21 #include <linux/module.h> 22 #include <linux/socket.h> 23 #include <linux/workqueue.h> 24 #include <linux/in.h> 25 #include <linux/sched/signal.h> 26 #include <linux/if_vlan.h> 27 28 #include <net/sock.h> 29 #include <net/tcp.h> 30 #include <net/smc.h> 31 #include <asm/ioctls.h> 32 33 #include "smc.h" 34 #include "smc_clc.h" 35 #include "smc_llc.h" 36 #include "smc_cdc.h" 37 #include "smc_core.h" 38 #include "smc_ib.h" 39 #include "smc_ism.h" 40 #include "smc_pnet.h" 41 #include "smc_tx.h" 42 #include "smc_rx.h" 43 #include "smc_close.h" 44 45 static DEFINE_MUTEX(smc_create_lgr_pending); /* serialize link group 46 * creation 47 */ 48 49 static void smc_tcp_listen_work(struct work_struct *); 50 static void smc_connect_work(struct work_struct *); 51 52 static void smc_set_keepalive(struct sock *sk, int val) 53 { 54 struct smc_sock *smc = smc_sk(sk); 55 56 smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val); 57 } 58 59 static struct smc_hashinfo smc_v4_hashinfo = { 60 .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock), 61 }; 62 63 static struct smc_hashinfo smc_v6_hashinfo = { 64 .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock), 65 }; 66 67 int smc_hash_sk(struct sock *sk) 68 { 69 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash; 70 struct hlist_head *head; 71 72 head = &h->ht; 73 74 write_lock_bh(&h->lock); 75 sk_add_node(sk, head); 76 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 77 write_unlock_bh(&h->lock); 78 79 return 0; 80 } 81 EXPORT_SYMBOL_GPL(smc_hash_sk); 82 83 void smc_unhash_sk(struct sock *sk) 84 { 85 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash; 86 87 write_lock_bh(&h->lock); 88 if (sk_del_node_init(sk)) 89 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); 90 write_unlock_bh(&h->lock); 91 } 92 EXPORT_SYMBOL_GPL(smc_unhash_sk); 93 94 struct proto smc_proto = { 95 .name = "SMC", 96 .owner = THIS_MODULE, 97 .keepalive = smc_set_keepalive, 98 .hash = smc_hash_sk, 99 .unhash = smc_unhash_sk, 100 .obj_size = sizeof(struct smc_sock), 101 .h.smc_hash = &smc_v4_hashinfo, 102 .slab_flags = SLAB_TYPESAFE_BY_RCU, 103 }; 104 EXPORT_SYMBOL_GPL(smc_proto); 105 106 struct proto smc_proto6 = { 107 .name = "SMC6", 108 .owner = THIS_MODULE, 109 .keepalive = smc_set_keepalive, 110 .hash = smc_hash_sk, 111 .unhash = smc_unhash_sk, 112 .obj_size = sizeof(struct smc_sock), 113 .h.smc_hash = &smc_v6_hashinfo, 114 .slab_flags = SLAB_TYPESAFE_BY_RCU, 115 }; 116 EXPORT_SYMBOL_GPL(smc_proto6); 117 118 static int smc_release(struct socket *sock) 119 { 120 struct sock *sk = sock->sk; 121 struct smc_sock *smc; 122 int rc = 0; 123 124 if (!sk) 125 goto out; 126 127 smc = smc_sk(sk); 128 129 /* cleanup for a dangling non-blocking connect */ 130 flush_work(&smc->connect_work); 131 kfree(smc->connect_info); 132 smc->connect_info = NULL; 133 134 if (sk->sk_state == SMC_LISTEN) 135 /* smc_close_non_accepted() is called and acquires 136 * sock lock for child sockets again 137 */ 138 lock_sock_nested(sk, SINGLE_DEPTH_NESTING); 139 else 140 lock_sock(sk); 141 142 if (!smc->use_fallback) { 143 rc = smc_close_active(smc); 144 sock_set_flag(sk, SOCK_DEAD); 145 sk->sk_shutdown |= SHUTDOWN_MASK; 146 } 147 if (smc->clcsock) { 148 sock_release(smc->clcsock); 149 smc->clcsock = NULL; 150 } 151 if (smc->use_fallback) { 152 if (sk->sk_state != SMC_LISTEN && sk->sk_state != SMC_INIT) 153 sock_put(sk); /* passive closing */ 154 sk->sk_state = SMC_CLOSED; 155 sk->sk_state_change(sk); 156 } 157 158 /* detach socket */ 159 sock_orphan(sk); 160 sock->sk = NULL; 161 if (!smc->use_fallback && sk->sk_state == SMC_CLOSED) 162 smc_conn_free(&smc->conn); 163 release_sock(sk); 164 165 sk->sk_prot->unhash(sk); 166 sock_put(sk); /* final sock_put */ 167 out: 168 return rc; 169 } 170 171 static void smc_destruct(struct sock *sk) 172 { 173 if (sk->sk_state != SMC_CLOSED) 174 return; 175 if (!sock_flag(sk, SOCK_DEAD)) 176 return; 177 178 sk_refcnt_debug_dec(sk); 179 } 180 181 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock, 182 int protocol) 183 { 184 struct smc_sock *smc; 185 struct proto *prot; 186 struct sock *sk; 187 188 prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto; 189 sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0); 190 if (!sk) 191 return NULL; 192 193 sock_init_data(sock, sk); /* sets sk_refcnt to 1 */ 194 sk->sk_state = SMC_INIT; 195 sk->sk_destruct = smc_destruct; 196 sk->sk_protocol = protocol; 197 smc = smc_sk(sk); 198 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work); 199 INIT_WORK(&smc->connect_work, smc_connect_work); 200 INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work); 201 INIT_LIST_HEAD(&smc->accept_q); 202 spin_lock_init(&smc->accept_q_lock); 203 spin_lock_init(&smc->conn.send_lock); 204 sk->sk_prot->hash(sk); 205 sk_refcnt_debug_inc(sk); 206 207 return sk; 208 } 209 210 static int smc_bind(struct socket *sock, struct sockaddr *uaddr, 211 int addr_len) 212 { 213 struct sockaddr_in *addr = (struct sockaddr_in *)uaddr; 214 struct sock *sk = sock->sk; 215 struct smc_sock *smc; 216 int rc; 217 218 smc = smc_sk(sk); 219 220 /* replicate tests from inet_bind(), to be safe wrt. future changes */ 221 rc = -EINVAL; 222 if (addr_len < sizeof(struct sockaddr_in)) 223 goto out; 224 225 rc = -EAFNOSUPPORT; 226 if (addr->sin_family != AF_INET && 227 addr->sin_family != AF_INET6 && 228 addr->sin_family != AF_UNSPEC) 229 goto out; 230 /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */ 231 if (addr->sin_family == AF_UNSPEC && 232 addr->sin_addr.s_addr != htonl(INADDR_ANY)) 233 goto out; 234 235 lock_sock(sk); 236 237 /* Check if socket is already active */ 238 rc = -EINVAL; 239 if (sk->sk_state != SMC_INIT) 240 goto out_rel; 241 242 smc->clcsock->sk->sk_reuse = sk->sk_reuse; 243 rc = kernel_bind(smc->clcsock, uaddr, addr_len); 244 245 out_rel: 246 release_sock(sk); 247 out: 248 return rc; 249 } 250 251 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk, 252 unsigned long mask) 253 { 254 /* options we don't get control via setsockopt for */ 255 nsk->sk_type = osk->sk_type; 256 nsk->sk_sndbuf = osk->sk_sndbuf; 257 nsk->sk_rcvbuf = osk->sk_rcvbuf; 258 nsk->sk_sndtimeo = osk->sk_sndtimeo; 259 nsk->sk_rcvtimeo = osk->sk_rcvtimeo; 260 nsk->sk_mark = osk->sk_mark; 261 nsk->sk_priority = osk->sk_priority; 262 nsk->sk_rcvlowat = osk->sk_rcvlowat; 263 nsk->sk_bound_dev_if = osk->sk_bound_dev_if; 264 nsk->sk_err = osk->sk_err; 265 266 nsk->sk_flags &= ~mask; 267 nsk->sk_flags |= osk->sk_flags & mask; 268 } 269 270 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \ 271 (1UL << SOCK_KEEPOPEN) | \ 272 (1UL << SOCK_LINGER) | \ 273 (1UL << SOCK_BROADCAST) | \ 274 (1UL << SOCK_TIMESTAMP) | \ 275 (1UL << SOCK_DBG) | \ 276 (1UL << SOCK_RCVTSTAMP) | \ 277 (1UL << SOCK_RCVTSTAMPNS) | \ 278 (1UL << SOCK_LOCALROUTE) | \ 279 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \ 280 (1UL << SOCK_RXQ_OVFL) | \ 281 (1UL << SOCK_WIFI_STATUS) | \ 282 (1UL << SOCK_NOFCS) | \ 283 (1UL << SOCK_FILTER_LOCKED)) 284 /* copy only relevant settings and flags of SOL_SOCKET level from smc to 285 * clc socket (since smc is not called for these options from net/core) 286 */ 287 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc) 288 { 289 smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC); 290 } 291 292 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \ 293 (1UL << SOCK_KEEPOPEN) | \ 294 (1UL << SOCK_LINGER) | \ 295 (1UL << SOCK_DBG)) 296 /* copy only settings and flags relevant for smc from clc to smc socket */ 297 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc) 298 { 299 smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC); 300 } 301 302 /* register a new rmb, optionally send confirm_rkey msg to register with peer */ 303 static int smc_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc, 304 bool conf_rkey) 305 { 306 /* register memory region for new rmb */ 307 if (smc_wr_reg_send(link, rmb_desc->mr_rx[SMC_SINGLE_LINK])) { 308 rmb_desc->regerr = 1; 309 return -EFAULT; 310 } 311 if (!conf_rkey) 312 return 0; 313 /* exchange confirm_rkey msg with peer */ 314 if (smc_llc_do_confirm_rkey(link, rmb_desc)) { 315 rmb_desc->regerr = 1; 316 return -EFAULT; 317 } 318 return 0; 319 } 320 321 static int smc_clnt_conf_first_link(struct smc_sock *smc) 322 { 323 struct net *net = sock_net(smc->clcsock->sk); 324 struct smc_link_group *lgr = smc->conn.lgr; 325 struct smc_link *link; 326 int rest; 327 int rc; 328 329 link = &lgr->lnk[SMC_SINGLE_LINK]; 330 /* receive CONFIRM LINK request from server over RoCE fabric */ 331 rest = wait_for_completion_interruptible_timeout( 332 &link->llc_confirm, 333 SMC_LLC_WAIT_FIRST_TIME); 334 if (rest <= 0) { 335 struct smc_clc_msg_decline dclc; 336 337 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), 338 SMC_CLC_DECLINE); 339 return rc; 340 } 341 342 if (link->llc_confirm_rc) 343 return SMC_CLC_DECL_RMBE_EC; 344 345 rc = smc_ib_modify_qp_rts(link); 346 if (rc) 347 return SMC_CLC_DECL_ERR_RDYLNK; 348 349 smc_wr_remember_qp_attr(link); 350 351 if (smc_reg_rmb(link, smc->conn.rmb_desc, false)) 352 return SMC_CLC_DECL_ERR_REGRMB; 353 354 /* send CONFIRM LINK response over RoCE fabric */ 355 rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP); 356 if (rc < 0) 357 return SMC_CLC_DECL_TIMEOUT_CL; 358 359 /* receive ADD LINK request from server over RoCE fabric */ 360 rest = wait_for_completion_interruptible_timeout(&link->llc_add, 361 SMC_LLC_WAIT_TIME); 362 if (rest <= 0) { 363 struct smc_clc_msg_decline dclc; 364 365 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), 366 SMC_CLC_DECLINE); 367 return rc; 368 } 369 370 /* send add link reject message, only one link supported for now */ 371 rc = smc_llc_send_add_link(link, 372 link->smcibdev->mac[link->ibport - 1], 373 link->gid, SMC_LLC_RESP); 374 if (rc < 0) 375 return SMC_CLC_DECL_TIMEOUT_AL; 376 377 smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time); 378 379 return 0; 380 } 381 382 static void smcr_conn_save_peer_info(struct smc_sock *smc, 383 struct smc_clc_msg_accept_confirm *clc) 384 { 385 int bufsize = smc_uncompress_bufsize(clc->rmbe_size); 386 387 smc->conn.peer_rmbe_idx = clc->rmbe_idx; 388 smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token); 389 smc->conn.peer_rmbe_size = bufsize; 390 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size); 391 smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1); 392 } 393 394 static void smcd_conn_save_peer_info(struct smc_sock *smc, 395 struct smc_clc_msg_accept_confirm *clc) 396 { 397 int bufsize = smc_uncompress_bufsize(clc->dmbe_size); 398 399 smc->conn.peer_rmbe_idx = clc->dmbe_idx; 400 smc->conn.peer_token = clc->token; 401 /* msg header takes up space in the buffer */ 402 smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg); 403 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size); 404 smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx; 405 } 406 407 static void smc_conn_save_peer_info(struct smc_sock *smc, 408 struct smc_clc_msg_accept_confirm *clc) 409 { 410 if (smc->conn.lgr->is_smcd) 411 smcd_conn_save_peer_info(smc, clc); 412 else 413 smcr_conn_save_peer_info(smc, clc); 414 } 415 416 static void smc_link_save_peer_info(struct smc_link *link, 417 struct smc_clc_msg_accept_confirm *clc) 418 { 419 link->peer_qpn = ntoh24(clc->qpn); 420 memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE); 421 memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac)); 422 link->peer_psn = ntoh24(clc->psn); 423 link->peer_mtu = clc->qp_mtu; 424 } 425 426 /* fall back during connect */ 427 static int smc_connect_fallback(struct smc_sock *smc, int reason_code) 428 { 429 smc->use_fallback = true; 430 smc->fallback_rsn = reason_code; 431 smc_copy_sock_settings_to_clc(smc); 432 if (smc->sk.sk_state == SMC_INIT) 433 smc->sk.sk_state = SMC_ACTIVE; 434 return 0; 435 } 436 437 /* decline and fall back during connect */ 438 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code) 439 { 440 int rc; 441 442 if (reason_code < 0) { /* error, fallback is not possible */ 443 if (smc->sk.sk_state == SMC_INIT) 444 sock_put(&smc->sk); /* passive closing */ 445 return reason_code; 446 } 447 if (reason_code != SMC_CLC_DECL_PEERDECL) { 448 rc = smc_clc_send_decline(smc, reason_code); 449 if (rc < 0) { 450 if (smc->sk.sk_state == SMC_INIT) 451 sock_put(&smc->sk); /* passive closing */ 452 return rc; 453 } 454 } 455 return smc_connect_fallback(smc, reason_code); 456 } 457 458 /* abort connecting */ 459 static int smc_connect_abort(struct smc_sock *smc, int reason_code, 460 int local_contact) 461 { 462 if (local_contact == SMC_FIRST_CONTACT) 463 smc_lgr_forget(smc->conn.lgr); 464 mutex_unlock(&smc_create_lgr_pending); 465 smc_conn_free(&smc->conn); 466 return reason_code; 467 } 468 469 /* check if there is a rdma device available for this connection. */ 470 /* called for connect and listen */ 471 static int smc_check_rdma(struct smc_sock *smc, struct smc_ib_device **ibdev, 472 u8 *ibport, unsigned short vlan_id, u8 gid[]) 473 { 474 int reason_code = 0; 475 476 /* PNET table look up: search active ib_device and port 477 * within same PNETID that also contains the ethernet device 478 * used for the internal TCP socket 479 */ 480 smc_pnet_find_roce_resource(smc->clcsock->sk, ibdev, ibport, vlan_id, 481 gid); 482 if (!(*ibdev)) 483 reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */ 484 485 return reason_code; 486 } 487 488 /* check if there is an ISM device available for this connection. */ 489 /* called for connect and listen */ 490 static int smc_check_ism(struct smc_sock *smc, struct smcd_dev **ismdev) 491 { 492 /* Find ISM device with same PNETID as connecting interface */ 493 smc_pnet_find_ism_resource(smc->clcsock->sk, ismdev); 494 if (!(*ismdev)) 495 return SMC_CLC_DECL_CNFERR; /* configuration error */ 496 return 0; 497 } 498 499 /* Check for VLAN ID and register it on ISM device just for CLC handshake */ 500 static int smc_connect_ism_vlan_setup(struct smc_sock *smc, 501 struct smcd_dev *ismdev, 502 unsigned short vlan_id) 503 { 504 if (vlan_id && smc_ism_get_vlan(ismdev, vlan_id)) 505 return SMC_CLC_DECL_CNFERR; 506 return 0; 507 } 508 509 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is 510 * used, the VLAN ID will be registered again during the connection setup. 511 */ 512 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, bool is_smcd, 513 struct smcd_dev *ismdev, 514 unsigned short vlan_id) 515 { 516 if (!is_smcd) 517 return 0; 518 if (vlan_id && smc_ism_put_vlan(ismdev, vlan_id)) 519 return SMC_CLC_DECL_CNFERR; 520 return 0; 521 } 522 523 /* CLC handshake during connect */ 524 static int smc_connect_clc(struct smc_sock *smc, int smc_type, 525 struct smc_clc_msg_accept_confirm *aclc, 526 struct smc_ib_device *ibdev, u8 ibport, 527 u8 gid[], struct smcd_dev *ismdev) 528 { 529 int rc = 0; 530 531 /* do inband token exchange */ 532 rc = smc_clc_send_proposal(smc, smc_type, ibdev, ibport, gid, ismdev); 533 if (rc) 534 return rc; 535 /* receive SMC Accept CLC message */ 536 return smc_clc_wait_msg(smc, aclc, sizeof(*aclc), SMC_CLC_ACCEPT); 537 } 538 539 /* setup for RDMA connection of client */ 540 static int smc_connect_rdma(struct smc_sock *smc, 541 struct smc_clc_msg_accept_confirm *aclc, 542 struct smc_ib_device *ibdev, u8 ibport) 543 { 544 int local_contact = SMC_FIRST_CONTACT; 545 struct smc_link *link; 546 int reason_code = 0; 547 548 mutex_lock(&smc_create_lgr_pending); 549 local_contact = smc_conn_create(smc, false, aclc->hdr.flag, ibdev, 550 ibport, &aclc->lcl, NULL, 0); 551 if (local_contact < 0) { 552 if (local_contact == -ENOMEM) 553 reason_code = SMC_CLC_DECL_MEM;/* insufficient memory*/ 554 else if (local_contact == -ENOLINK) 555 reason_code = SMC_CLC_DECL_SYNCERR; /* synchr. error */ 556 else 557 reason_code = SMC_CLC_DECL_INTERR; /* other error */ 558 return smc_connect_abort(smc, reason_code, 0); 559 } 560 link = &smc->conn.lgr->lnk[SMC_SINGLE_LINK]; 561 562 smc_conn_save_peer_info(smc, aclc); 563 564 /* create send buffer and rmb */ 565 if (smc_buf_create(smc, false)) 566 return smc_connect_abort(smc, SMC_CLC_DECL_MEM, local_contact); 567 568 if (local_contact == SMC_FIRST_CONTACT) 569 smc_link_save_peer_info(link, aclc); 570 571 if (smc_rmb_rtoken_handling(&smc->conn, aclc)) 572 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RTOK, 573 local_contact); 574 575 smc_close_init(smc); 576 smc_rx_init(smc); 577 578 if (local_contact == SMC_FIRST_CONTACT) { 579 if (smc_ib_ready_link(link)) 580 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RDYLNK, 581 local_contact); 582 } else { 583 if (!smc->conn.rmb_desc->reused && 584 smc_reg_rmb(link, smc->conn.rmb_desc, true)) 585 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_REGRMB, 586 local_contact); 587 } 588 smc_rmb_sync_sg_for_device(&smc->conn); 589 590 reason_code = smc_clc_send_confirm(smc); 591 if (reason_code) 592 return smc_connect_abort(smc, reason_code, local_contact); 593 594 smc_tx_init(smc); 595 596 if (local_contact == SMC_FIRST_CONTACT) { 597 /* QP confirmation over RoCE fabric */ 598 reason_code = smc_clnt_conf_first_link(smc); 599 if (reason_code) 600 return smc_connect_abort(smc, reason_code, 601 local_contact); 602 } 603 mutex_unlock(&smc_create_lgr_pending); 604 605 smc_copy_sock_settings_to_clc(smc); 606 if (smc->sk.sk_state == SMC_INIT) 607 smc->sk.sk_state = SMC_ACTIVE; 608 609 return 0; 610 } 611 612 /* setup for ISM connection of client */ 613 static int smc_connect_ism(struct smc_sock *smc, 614 struct smc_clc_msg_accept_confirm *aclc, 615 struct smcd_dev *ismdev) 616 { 617 int local_contact = SMC_FIRST_CONTACT; 618 int rc = 0; 619 620 mutex_lock(&smc_create_lgr_pending); 621 local_contact = smc_conn_create(smc, true, aclc->hdr.flag, NULL, 0, 622 NULL, ismdev, aclc->gid); 623 if (local_contact < 0) 624 return smc_connect_abort(smc, SMC_CLC_DECL_MEM, 0); 625 626 /* Create send and receive buffers */ 627 if (smc_buf_create(smc, true)) 628 return smc_connect_abort(smc, SMC_CLC_DECL_MEM, local_contact); 629 630 smc_conn_save_peer_info(smc, aclc); 631 smc_close_init(smc); 632 smc_rx_init(smc); 633 smc_tx_init(smc); 634 635 rc = smc_clc_send_confirm(smc); 636 if (rc) 637 return smc_connect_abort(smc, rc, local_contact); 638 mutex_unlock(&smc_create_lgr_pending); 639 640 smc_copy_sock_settings_to_clc(smc); 641 if (smc->sk.sk_state == SMC_INIT) 642 smc->sk.sk_state = SMC_ACTIVE; 643 644 return 0; 645 } 646 647 /* perform steps before actually connecting */ 648 static int __smc_connect(struct smc_sock *smc) 649 { 650 bool ism_supported = false, rdma_supported = false; 651 struct smc_clc_msg_accept_confirm aclc; 652 struct smc_ib_device *ibdev; 653 struct smcd_dev *ismdev; 654 u8 gid[SMC_GID_SIZE]; 655 unsigned short vlan; 656 int smc_type; 657 int rc = 0; 658 u8 ibport; 659 660 sock_hold(&smc->sk); /* sock put in passive closing */ 661 662 if (smc->use_fallback) 663 return smc_connect_fallback(smc, smc->fallback_rsn); 664 665 /* if peer has not signalled SMC-capability, fall back */ 666 if (!tcp_sk(smc->clcsock->sk)->syn_smc) 667 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC); 668 669 /* IPSec connections opt out of SMC-R optimizations */ 670 if (using_ipsec(smc)) 671 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC); 672 673 /* check for VLAN ID */ 674 if (smc_vlan_by_tcpsk(smc->clcsock, &vlan)) 675 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_CNFERR); 676 677 /* check if there is an ism device available */ 678 if (!smc_check_ism(smc, &ismdev) && 679 !smc_connect_ism_vlan_setup(smc, ismdev, vlan)) { 680 /* ISM is supported for this connection */ 681 ism_supported = true; 682 smc_type = SMC_TYPE_D; 683 } 684 685 /* check if there is a rdma device available */ 686 if (!smc_check_rdma(smc, &ibdev, &ibport, vlan, gid)) { 687 /* RDMA is supported for this connection */ 688 rdma_supported = true; 689 if (ism_supported) 690 smc_type = SMC_TYPE_B; /* both */ 691 else 692 smc_type = SMC_TYPE_R; /* only RDMA */ 693 } 694 695 /* if neither ISM nor RDMA are supported, fallback */ 696 if (!rdma_supported && !ism_supported) 697 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_NOSMCDEV); 698 699 /* perform CLC handshake */ 700 rc = smc_connect_clc(smc, smc_type, &aclc, ibdev, ibport, gid, ismdev); 701 if (rc) { 702 smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan); 703 return smc_connect_decline_fallback(smc, rc); 704 } 705 706 /* depending on previous steps, connect using rdma or ism */ 707 if (rdma_supported && aclc.hdr.path == SMC_TYPE_R) 708 rc = smc_connect_rdma(smc, &aclc, ibdev, ibport); 709 else if (ism_supported && aclc.hdr.path == SMC_TYPE_D) 710 rc = smc_connect_ism(smc, &aclc, ismdev); 711 else 712 rc = SMC_CLC_DECL_MODEUNSUPP; 713 if (rc) { 714 smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan); 715 return smc_connect_decline_fallback(smc, rc); 716 } 717 718 smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan); 719 return 0; 720 } 721 722 static void smc_connect_work(struct work_struct *work) 723 { 724 struct smc_sock *smc = container_of(work, struct smc_sock, 725 connect_work); 726 int rc; 727 728 lock_sock(&smc->sk); 729 rc = kernel_connect(smc->clcsock, &smc->connect_info->addr, 730 smc->connect_info->alen, smc->connect_info->flags); 731 if (smc->clcsock->sk->sk_err) { 732 smc->sk.sk_err = smc->clcsock->sk->sk_err; 733 goto out; 734 } 735 if (rc < 0) { 736 smc->sk.sk_err = -rc; 737 goto out; 738 } 739 740 rc = __smc_connect(smc); 741 if (rc < 0) 742 smc->sk.sk_err = -rc; 743 744 out: 745 if (smc->sk.sk_err) 746 smc->sk.sk_state_change(&smc->sk); 747 else 748 smc->sk.sk_write_space(&smc->sk); 749 kfree(smc->connect_info); 750 smc->connect_info = NULL; 751 release_sock(&smc->sk); 752 } 753 754 static int smc_connect(struct socket *sock, struct sockaddr *addr, 755 int alen, int flags) 756 { 757 struct sock *sk = sock->sk; 758 struct smc_sock *smc; 759 int rc = -EINVAL; 760 761 smc = smc_sk(sk); 762 763 /* separate smc parameter checking to be safe */ 764 if (alen < sizeof(addr->sa_family)) 765 goto out_err; 766 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6) 767 goto out_err; 768 769 lock_sock(sk); 770 switch (sk->sk_state) { 771 default: 772 goto out; 773 case SMC_ACTIVE: 774 rc = -EISCONN; 775 goto out; 776 case SMC_INIT: 777 rc = 0; 778 break; 779 } 780 781 smc_copy_sock_settings_to_clc(smc); 782 tcp_sk(smc->clcsock->sk)->syn_smc = 1; 783 if (flags & O_NONBLOCK) { 784 if (smc->connect_info) { 785 rc = -EALREADY; 786 goto out; 787 } 788 smc->connect_info = kzalloc(alen + 2 * sizeof(int), GFP_KERNEL); 789 if (!smc->connect_info) { 790 rc = -ENOMEM; 791 goto out; 792 } 793 smc->connect_info->alen = alen; 794 smc->connect_info->flags = flags ^ O_NONBLOCK; 795 memcpy(&smc->connect_info->addr, addr, alen); 796 schedule_work(&smc->connect_work); 797 rc = -EINPROGRESS; 798 } else { 799 rc = kernel_connect(smc->clcsock, addr, alen, flags); 800 if (rc) 801 goto out; 802 803 rc = __smc_connect(smc); 804 if (rc < 0) 805 goto out; 806 else 807 rc = 0; /* success cases including fallback */ 808 } 809 810 out: 811 release_sock(sk); 812 out_err: 813 return rc; 814 } 815 816 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc) 817 { 818 struct socket *new_clcsock = NULL; 819 struct sock *lsk = &lsmc->sk; 820 struct sock *new_sk; 821 int rc; 822 823 release_sock(lsk); 824 new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol); 825 if (!new_sk) { 826 rc = -ENOMEM; 827 lsk->sk_err = ENOMEM; 828 *new_smc = NULL; 829 lock_sock(lsk); 830 goto out; 831 } 832 *new_smc = smc_sk(new_sk); 833 834 rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0); 835 lock_sock(lsk); 836 if (rc < 0) 837 lsk->sk_err = -rc; 838 if (rc < 0 || lsk->sk_state == SMC_CLOSED) { 839 if (new_clcsock) 840 sock_release(new_clcsock); 841 new_sk->sk_state = SMC_CLOSED; 842 sock_set_flag(new_sk, SOCK_DEAD); 843 new_sk->sk_prot->unhash(new_sk); 844 sock_put(new_sk); /* final */ 845 *new_smc = NULL; 846 goto out; 847 } 848 849 (*new_smc)->clcsock = new_clcsock; 850 out: 851 return rc; 852 } 853 854 /* add a just created sock to the accept queue of the listen sock as 855 * candidate for a following socket accept call from user space 856 */ 857 static void smc_accept_enqueue(struct sock *parent, struct sock *sk) 858 { 859 struct smc_sock *par = smc_sk(parent); 860 861 sock_hold(sk); /* sock_put in smc_accept_unlink () */ 862 spin_lock(&par->accept_q_lock); 863 list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q); 864 spin_unlock(&par->accept_q_lock); 865 sk_acceptq_added(parent); 866 } 867 868 /* remove a socket from the accept queue of its parental listening socket */ 869 static void smc_accept_unlink(struct sock *sk) 870 { 871 struct smc_sock *par = smc_sk(sk)->listen_smc; 872 873 spin_lock(&par->accept_q_lock); 874 list_del_init(&smc_sk(sk)->accept_q); 875 spin_unlock(&par->accept_q_lock); 876 sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk); 877 sock_put(sk); /* sock_hold in smc_accept_enqueue */ 878 } 879 880 /* remove a sock from the accept queue to bind it to a new socket created 881 * for a socket accept call from user space 882 */ 883 struct sock *smc_accept_dequeue(struct sock *parent, 884 struct socket *new_sock) 885 { 886 struct smc_sock *isk, *n; 887 struct sock *new_sk; 888 889 list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) { 890 new_sk = (struct sock *)isk; 891 892 smc_accept_unlink(new_sk); 893 if (new_sk->sk_state == SMC_CLOSED) { 894 if (isk->clcsock) { 895 sock_release(isk->clcsock); 896 isk->clcsock = NULL; 897 } 898 new_sk->sk_prot->unhash(new_sk); 899 sock_put(new_sk); /* final */ 900 continue; 901 } 902 if (new_sock) 903 sock_graft(new_sk, new_sock); 904 return new_sk; 905 } 906 return NULL; 907 } 908 909 /* clean up for a created but never accepted sock */ 910 void smc_close_non_accepted(struct sock *sk) 911 { 912 struct smc_sock *smc = smc_sk(sk); 913 914 lock_sock(sk); 915 if (!sk->sk_lingertime) 916 /* wait for peer closing */ 917 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT; 918 if (!smc->use_fallback) { 919 smc_close_active(smc); 920 sock_set_flag(sk, SOCK_DEAD); 921 sk->sk_shutdown |= SHUTDOWN_MASK; 922 } 923 if (smc->clcsock) { 924 struct socket *tcp; 925 926 tcp = smc->clcsock; 927 smc->clcsock = NULL; 928 sock_release(tcp); 929 } 930 if (smc->use_fallback) { 931 sock_put(sk); /* passive closing */ 932 sk->sk_state = SMC_CLOSED; 933 } else { 934 if (sk->sk_state == SMC_CLOSED) 935 smc_conn_free(&smc->conn); 936 } 937 release_sock(sk); 938 sk->sk_prot->unhash(sk); 939 sock_put(sk); /* final sock_put */ 940 } 941 942 static int smc_serv_conf_first_link(struct smc_sock *smc) 943 { 944 struct net *net = sock_net(smc->clcsock->sk); 945 struct smc_link_group *lgr = smc->conn.lgr; 946 struct smc_link *link; 947 int rest; 948 int rc; 949 950 link = &lgr->lnk[SMC_SINGLE_LINK]; 951 952 if (smc_reg_rmb(link, smc->conn.rmb_desc, false)) 953 return SMC_CLC_DECL_ERR_REGRMB; 954 955 /* send CONFIRM LINK request to client over the RoCE fabric */ 956 rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ); 957 if (rc < 0) 958 return SMC_CLC_DECL_TIMEOUT_CL; 959 960 /* receive CONFIRM LINK response from client over the RoCE fabric */ 961 rest = wait_for_completion_interruptible_timeout( 962 &link->llc_confirm_resp, 963 SMC_LLC_WAIT_FIRST_TIME); 964 if (rest <= 0) { 965 struct smc_clc_msg_decline dclc; 966 967 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), 968 SMC_CLC_DECLINE); 969 return rc; 970 } 971 972 if (link->llc_confirm_resp_rc) 973 return SMC_CLC_DECL_RMBE_EC; 974 975 /* send ADD LINK request to client over the RoCE fabric */ 976 rc = smc_llc_send_add_link(link, 977 link->smcibdev->mac[link->ibport - 1], 978 link->gid, SMC_LLC_REQ); 979 if (rc < 0) 980 return SMC_CLC_DECL_TIMEOUT_AL; 981 982 /* receive ADD LINK response from client over the RoCE fabric */ 983 rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp, 984 SMC_LLC_WAIT_TIME); 985 if (rest <= 0) { 986 struct smc_clc_msg_decline dclc; 987 988 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), 989 SMC_CLC_DECLINE); 990 return rc; 991 } 992 993 smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time); 994 995 return 0; 996 } 997 998 /* listen worker: finish */ 999 static void smc_listen_out(struct smc_sock *new_smc) 1000 { 1001 struct smc_sock *lsmc = new_smc->listen_smc; 1002 struct sock *newsmcsk = &new_smc->sk; 1003 1004 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING); 1005 if (lsmc->sk.sk_state == SMC_LISTEN) { 1006 smc_accept_enqueue(&lsmc->sk, newsmcsk); 1007 } else { /* no longer listening */ 1008 smc_close_non_accepted(newsmcsk); 1009 } 1010 release_sock(&lsmc->sk); 1011 1012 /* Wake up accept */ 1013 lsmc->sk.sk_data_ready(&lsmc->sk); 1014 sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */ 1015 } 1016 1017 /* listen worker: finish in state connected */ 1018 static void smc_listen_out_connected(struct smc_sock *new_smc) 1019 { 1020 struct sock *newsmcsk = &new_smc->sk; 1021 1022 sk_refcnt_debug_inc(newsmcsk); 1023 if (newsmcsk->sk_state == SMC_INIT) 1024 newsmcsk->sk_state = SMC_ACTIVE; 1025 1026 smc_listen_out(new_smc); 1027 } 1028 1029 /* listen worker: finish in error state */ 1030 static void smc_listen_out_err(struct smc_sock *new_smc) 1031 { 1032 struct sock *newsmcsk = &new_smc->sk; 1033 1034 if (newsmcsk->sk_state == SMC_INIT) 1035 sock_put(&new_smc->sk); /* passive closing */ 1036 newsmcsk->sk_state = SMC_CLOSED; 1037 smc_conn_free(&new_smc->conn); 1038 1039 smc_listen_out(new_smc); 1040 } 1041 1042 /* listen worker: decline and fall back if possible */ 1043 static void smc_listen_decline(struct smc_sock *new_smc, int reason_code, 1044 int local_contact) 1045 { 1046 /* RDMA setup failed, switch back to TCP */ 1047 if (local_contact == SMC_FIRST_CONTACT) 1048 smc_lgr_forget(new_smc->conn.lgr); 1049 if (reason_code < 0) { /* error, no fallback possible */ 1050 smc_listen_out_err(new_smc); 1051 return; 1052 } 1053 smc_conn_free(&new_smc->conn); 1054 new_smc->use_fallback = true; 1055 new_smc->fallback_rsn = reason_code; 1056 if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) { 1057 if (smc_clc_send_decline(new_smc, reason_code) < 0) { 1058 smc_listen_out_err(new_smc); 1059 return; 1060 } 1061 } 1062 smc_listen_out_connected(new_smc); 1063 } 1064 1065 /* listen worker: check prefixes */ 1066 static int smc_listen_rdma_check(struct smc_sock *new_smc, 1067 struct smc_clc_msg_proposal *pclc) 1068 { 1069 struct smc_clc_msg_proposal_prefix *pclc_prfx; 1070 struct socket *newclcsock = new_smc->clcsock; 1071 1072 pclc_prfx = smc_clc_proposal_get_prefix(pclc); 1073 if (smc_clc_prfx_match(newclcsock, pclc_prfx)) 1074 return SMC_CLC_DECL_CNFERR; 1075 1076 return 0; 1077 } 1078 1079 /* listen worker: initialize connection and buffers */ 1080 static int smc_listen_rdma_init(struct smc_sock *new_smc, 1081 struct smc_clc_msg_proposal *pclc, 1082 struct smc_ib_device *ibdev, u8 ibport, 1083 int *local_contact) 1084 { 1085 /* allocate connection / link group */ 1086 *local_contact = smc_conn_create(new_smc, false, 0, ibdev, ibport, 1087 &pclc->lcl, NULL, 0); 1088 if (*local_contact < 0) { 1089 if (*local_contact == -ENOMEM) 1090 return SMC_CLC_DECL_MEM;/* insufficient memory*/ 1091 return SMC_CLC_DECL_INTERR; /* other error */ 1092 } 1093 1094 /* create send buffer and rmb */ 1095 if (smc_buf_create(new_smc, false)) 1096 return SMC_CLC_DECL_MEM; 1097 1098 return 0; 1099 } 1100 1101 /* listen worker: initialize connection and buffers for SMC-D */ 1102 static int smc_listen_ism_init(struct smc_sock *new_smc, 1103 struct smc_clc_msg_proposal *pclc, 1104 struct smcd_dev *ismdev, 1105 int *local_contact) 1106 { 1107 struct smc_clc_msg_smcd *pclc_smcd; 1108 1109 pclc_smcd = smc_get_clc_msg_smcd(pclc); 1110 *local_contact = smc_conn_create(new_smc, true, 0, NULL, 0, NULL, 1111 ismdev, pclc_smcd->gid); 1112 if (*local_contact < 0) { 1113 if (*local_contact == -ENOMEM) 1114 return SMC_CLC_DECL_MEM;/* insufficient memory*/ 1115 return SMC_CLC_DECL_INTERR; /* other error */ 1116 } 1117 1118 /* Check if peer can be reached via ISM device */ 1119 if (smc_ism_cantalk(new_smc->conn.lgr->peer_gid, 1120 new_smc->conn.lgr->vlan_id, 1121 new_smc->conn.lgr->smcd)) { 1122 if (*local_contact == SMC_FIRST_CONTACT) 1123 smc_lgr_forget(new_smc->conn.lgr); 1124 smc_conn_free(&new_smc->conn); 1125 return SMC_CLC_DECL_CNFERR; 1126 } 1127 1128 /* Create send and receive buffers */ 1129 if (smc_buf_create(new_smc, true)) { 1130 if (*local_contact == SMC_FIRST_CONTACT) 1131 smc_lgr_forget(new_smc->conn.lgr); 1132 smc_conn_free(&new_smc->conn); 1133 return SMC_CLC_DECL_MEM; 1134 } 1135 1136 return 0; 1137 } 1138 1139 /* listen worker: register buffers */ 1140 static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact) 1141 { 1142 struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK]; 1143 1144 if (local_contact != SMC_FIRST_CONTACT) { 1145 if (!new_smc->conn.rmb_desc->reused) { 1146 if (smc_reg_rmb(link, new_smc->conn.rmb_desc, true)) 1147 return SMC_CLC_DECL_ERR_REGRMB; 1148 } 1149 } 1150 smc_rmb_sync_sg_for_device(&new_smc->conn); 1151 1152 return 0; 1153 } 1154 1155 /* listen worker: finish RDMA setup */ 1156 static int smc_listen_rdma_finish(struct smc_sock *new_smc, 1157 struct smc_clc_msg_accept_confirm *cclc, 1158 int local_contact) 1159 { 1160 struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK]; 1161 int reason_code = 0; 1162 1163 if (local_contact == SMC_FIRST_CONTACT) 1164 smc_link_save_peer_info(link, cclc); 1165 1166 if (smc_rmb_rtoken_handling(&new_smc->conn, cclc)) { 1167 reason_code = SMC_CLC_DECL_ERR_RTOK; 1168 goto decline; 1169 } 1170 1171 if (local_contact == SMC_FIRST_CONTACT) { 1172 if (smc_ib_ready_link(link)) { 1173 reason_code = SMC_CLC_DECL_ERR_RDYLNK; 1174 goto decline; 1175 } 1176 /* QP confirmation over RoCE fabric */ 1177 reason_code = smc_serv_conf_first_link(new_smc); 1178 if (reason_code) 1179 goto decline; 1180 } 1181 return 0; 1182 1183 decline: 1184 mutex_unlock(&smc_create_lgr_pending); 1185 smc_listen_decline(new_smc, reason_code, local_contact); 1186 return reason_code; 1187 } 1188 1189 /* setup for RDMA connection of server */ 1190 static void smc_listen_work(struct work_struct *work) 1191 { 1192 struct smc_sock *new_smc = container_of(work, struct smc_sock, 1193 smc_listen_work); 1194 struct socket *newclcsock = new_smc->clcsock; 1195 struct smc_clc_msg_accept_confirm cclc; 1196 struct smc_clc_msg_proposal *pclc; 1197 struct smc_ib_device *ibdev; 1198 bool ism_supported = false; 1199 struct smcd_dev *ismdev; 1200 u8 buf[SMC_CLC_MAX_LEN]; 1201 int local_contact = 0; 1202 unsigned short vlan; 1203 int reason_code = 0; 1204 int rc = 0; 1205 u8 ibport; 1206 1207 if (new_smc->use_fallback) { 1208 smc_listen_out_connected(new_smc); 1209 return; 1210 } 1211 1212 /* check if peer is smc capable */ 1213 if (!tcp_sk(newclcsock->sk)->syn_smc) { 1214 new_smc->use_fallback = true; 1215 new_smc->fallback_rsn = SMC_CLC_DECL_PEERNOSMC; 1216 smc_listen_out_connected(new_smc); 1217 return; 1218 } 1219 1220 /* do inband token exchange - 1221 * wait for and receive SMC Proposal CLC message 1222 */ 1223 pclc = (struct smc_clc_msg_proposal *)&buf; 1224 reason_code = smc_clc_wait_msg(new_smc, pclc, SMC_CLC_MAX_LEN, 1225 SMC_CLC_PROPOSAL); 1226 if (reason_code) { 1227 smc_listen_decline(new_smc, reason_code, 0); 1228 return; 1229 } 1230 1231 /* IPSec connections opt out of SMC-R optimizations */ 1232 if (using_ipsec(new_smc)) { 1233 smc_listen_decline(new_smc, SMC_CLC_DECL_IPSEC, 0); 1234 return; 1235 } 1236 1237 mutex_lock(&smc_create_lgr_pending); 1238 smc_close_init(new_smc); 1239 smc_rx_init(new_smc); 1240 smc_tx_init(new_smc); 1241 1242 /* check if ISM is available */ 1243 if ((pclc->hdr.path == SMC_TYPE_D || pclc->hdr.path == SMC_TYPE_B) && 1244 !smc_check_ism(new_smc, &ismdev) && 1245 !smc_listen_ism_init(new_smc, pclc, ismdev, &local_contact)) { 1246 ism_supported = true; 1247 } 1248 1249 /* check if RDMA is available */ 1250 if (!ism_supported && 1251 ((pclc->hdr.path != SMC_TYPE_R && pclc->hdr.path != SMC_TYPE_B) || 1252 smc_vlan_by_tcpsk(new_smc->clcsock, &vlan) || 1253 smc_check_rdma(new_smc, &ibdev, &ibport, vlan, NULL) || 1254 smc_listen_rdma_check(new_smc, pclc) || 1255 smc_listen_rdma_init(new_smc, pclc, ibdev, ibport, 1256 &local_contact) || 1257 smc_listen_rdma_reg(new_smc, local_contact))) { 1258 /* SMC not supported, decline */ 1259 mutex_unlock(&smc_create_lgr_pending); 1260 smc_listen_decline(new_smc, SMC_CLC_DECL_MODEUNSUPP, 1261 local_contact); 1262 return; 1263 } 1264 1265 /* send SMC Accept CLC message */ 1266 rc = smc_clc_send_accept(new_smc, local_contact); 1267 if (rc) { 1268 mutex_unlock(&smc_create_lgr_pending); 1269 smc_listen_decline(new_smc, rc, local_contact); 1270 return; 1271 } 1272 1273 /* receive SMC Confirm CLC message */ 1274 reason_code = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc), 1275 SMC_CLC_CONFIRM); 1276 if (reason_code) { 1277 mutex_unlock(&smc_create_lgr_pending); 1278 smc_listen_decline(new_smc, reason_code, local_contact); 1279 return; 1280 } 1281 1282 /* finish worker */ 1283 if (!ism_supported) { 1284 if (smc_listen_rdma_finish(new_smc, &cclc, local_contact)) 1285 return; 1286 } 1287 smc_conn_save_peer_info(new_smc, &cclc); 1288 mutex_unlock(&smc_create_lgr_pending); 1289 smc_listen_out_connected(new_smc); 1290 } 1291 1292 static void smc_tcp_listen_work(struct work_struct *work) 1293 { 1294 struct smc_sock *lsmc = container_of(work, struct smc_sock, 1295 tcp_listen_work); 1296 struct sock *lsk = &lsmc->sk; 1297 struct smc_sock *new_smc; 1298 int rc = 0; 1299 1300 lock_sock(lsk); 1301 while (lsk->sk_state == SMC_LISTEN) { 1302 rc = smc_clcsock_accept(lsmc, &new_smc); 1303 if (rc) 1304 goto out; 1305 if (!new_smc) 1306 continue; 1307 1308 new_smc->listen_smc = lsmc; 1309 new_smc->use_fallback = lsmc->use_fallback; 1310 new_smc->fallback_rsn = lsmc->fallback_rsn; 1311 sock_hold(lsk); /* sock_put in smc_listen_work */ 1312 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work); 1313 smc_copy_sock_settings_to_smc(new_smc); 1314 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf; 1315 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf; 1316 sock_hold(&new_smc->sk); /* sock_put in passive closing */ 1317 if (!schedule_work(&new_smc->smc_listen_work)) 1318 sock_put(&new_smc->sk); 1319 } 1320 1321 out: 1322 release_sock(lsk); 1323 sock_put(&lsmc->sk); /* sock_hold in smc_listen */ 1324 } 1325 1326 static int smc_listen(struct socket *sock, int backlog) 1327 { 1328 struct sock *sk = sock->sk; 1329 struct smc_sock *smc; 1330 int rc; 1331 1332 smc = smc_sk(sk); 1333 lock_sock(sk); 1334 1335 rc = -EINVAL; 1336 if ((sk->sk_state != SMC_INIT) && (sk->sk_state != SMC_LISTEN)) 1337 goto out; 1338 1339 rc = 0; 1340 if (sk->sk_state == SMC_LISTEN) { 1341 sk->sk_max_ack_backlog = backlog; 1342 goto out; 1343 } 1344 /* some socket options are handled in core, so we could not apply 1345 * them to the clc socket -- copy smc socket options to clc socket 1346 */ 1347 smc_copy_sock_settings_to_clc(smc); 1348 if (!smc->use_fallback) 1349 tcp_sk(smc->clcsock->sk)->syn_smc = 1; 1350 1351 rc = kernel_listen(smc->clcsock, backlog); 1352 if (rc) 1353 goto out; 1354 sk->sk_max_ack_backlog = backlog; 1355 sk->sk_ack_backlog = 0; 1356 sk->sk_state = SMC_LISTEN; 1357 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work); 1358 sock_hold(sk); /* sock_hold in tcp_listen_worker */ 1359 if (!schedule_work(&smc->tcp_listen_work)) 1360 sock_put(sk); 1361 1362 out: 1363 release_sock(sk); 1364 return rc; 1365 } 1366 1367 static int smc_accept(struct socket *sock, struct socket *new_sock, 1368 int flags, bool kern) 1369 { 1370 struct sock *sk = sock->sk, *nsk; 1371 DECLARE_WAITQUEUE(wait, current); 1372 struct smc_sock *lsmc; 1373 long timeo; 1374 int rc = 0; 1375 1376 lsmc = smc_sk(sk); 1377 sock_hold(sk); /* sock_put below */ 1378 lock_sock(sk); 1379 1380 if (lsmc->sk.sk_state != SMC_LISTEN) { 1381 rc = -EINVAL; 1382 release_sock(sk); 1383 goto out; 1384 } 1385 1386 /* Wait for an incoming connection */ 1387 timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); 1388 add_wait_queue_exclusive(sk_sleep(sk), &wait); 1389 while (!(nsk = smc_accept_dequeue(sk, new_sock))) { 1390 set_current_state(TASK_INTERRUPTIBLE); 1391 if (!timeo) { 1392 rc = -EAGAIN; 1393 break; 1394 } 1395 release_sock(sk); 1396 timeo = schedule_timeout(timeo); 1397 /* wakeup by sk_data_ready in smc_listen_work() */ 1398 sched_annotate_sleep(); 1399 lock_sock(sk); 1400 if (signal_pending(current)) { 1401 rc = sock_intr_errno(timeo); 1402 break; 1403 } 1404 } 1405 set_current_state(TASK_RUNNING); 1406 remove_wait_queue(sk_sleep(sk), &wait); 1407 1408 if (!rc) 1409 rc = sock_error(nsk); 1410 release_sock(sk); 1411 if (rc) 1412 goto out; 1413 1414 if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) { 1415 /* wait till data arrives on the socket */ 1416 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept * 1417 MSEC_PER_SEC); 1418 if (smc_sk(nsk)->use_fallback) { 1419 struct sock *clcsk = smc_sk(nsk)->clcsock->sk; 1420 1421 lock_sock(clcsk); 1422 if (skb_queue_empty(&clcsk->sk_receive_queue)) 1423 sk_wait_data(clcsk, &timeo, NULL); 1424 release_sock(clcsk); 1425 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) { 1426 lock_sock(nsk); 1427 smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available); 1428 release_sock(nsk); 1429 } 1430 } 1431 1432 out: 1433 sock_put(sk); /* sock_hold above */ 1434 return rc; 1435 } 1436 1437 static int smc_getname(struct socket *sock, struct sockaddr *addr, 1438 int peer) 1439 { 1440 struct smc_sock *smc; 1441 1442 if (peer && (sock->sk->sk_state != SMC_ACTIVE) && 1443 (sock->sk->sk_state != SMC_APPCLOSEWAIT1)) 1444 return -ENOTCONN; 1445 1446 smc = smc_sk(sock->sk); 1447 1448 return smc->clcsock->ops->getname(smc->clcsock, addr, peer); 1449 } 1450 1451 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) 1452 { 1453 struct sock *sk = sock->sk; 1454 struct smc_sock *smc; 1455 int rc = -EPIPE; 1456 1457 smc = smc_sk(sk); 1458 lock_sock(sk); 1459 if ((sk->sk_state != SMC_ACTIVE) && 1460 (sk->sk_state != SMC_APPCLOSEWAIT1) && 1461 (sk->sk_state != SMC_INIT)) 1462 goto out; 1463 1464 if (msg->msg_flags & MSG_FASTOPEN) { 1465 if (sk->sk_state == SMC_INIT) { 1466 smc->use_fallback = true; 1467 smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP; 1468 } else { 1469 rc = -EINVAL; 1470 goto out; 1471 } 1472 } 1473 1474 if (smc->use_fallback) 1475 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len); 1476 else 1477 rc = smc_tx_sendmsg(smc, msg, len); 1478 out: 1479 release_sock(sk); 1480 return rc; 1481 } 1482 1483 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, 1484 int flags) 1485 { 1486 struct sock *sk = sock->sk; 1487 struct smc_sock *smc; 1488 int rc = -ENOTCONN; 1489 1490 smc = smc_sk(sk); 1491 lock_sock(sk); 1492 if ((sk->sk_state == SMC_INIT) || 1493 (sk->sk_state == SMC_LISTEN) || 1494 (sk->sk_state == SMC_CLOSED)) 1495 goto out; 1496 1497 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) { 1498 rc = 0; 1499 goto out; 1500 } 1501 1502 if (smc->use_fallback) { 1503 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags); 1504 } else { 1505 msg->msg_namelen = 0; 1506 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags); 1507 } 1508 1509 out: 1510 release_sock(sk); 1511 return rc; 1512 } 1513 1514 static __poll_t smc_accept_poll(struct sock *parent) 1515 { 1516 struct smc_sock *isk = smc_sk(parent); 1517 __poll_t mask = 0; 1518 1519 spin_lock(&isk->accept_q_lock); 1520 if (!list_empty(&isk->accept_q)) 1521 mask = EPOLLIN | EPOLLRDNORM; 1522 spin_unlock(&isk->accept_q_lock); 1523 1524 return mask; 1525 } 1526 1527 static __poll_t smc_poll(struct file *file, struct socket *sock, 1528 poll_table *wait) 1529 { 1530 struct sock *sk = sock->sk; 1531 __poll_t mask = 0; 1532 struct smc_sock *smc; 1533 1534 if (!sk) 1535 return EPOLLNVAL; 1536 1537 smc = smc_sk(sock->sk); 1538 if (smc->use_fallback) { 1539 /* delegate to CLC child sock */ 1540 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait); 1541 sk->sk_err = smc->clcsock->sk->sk_err; 1542 if (sk->sk_err) 1543 mask |= EPOLLERR; 1544 } else { 1545 if (sk->sk_state != SMC_CLOSED) 1546 sock_poll_wait(file, sock, wait); 1547 if (sk->sk_err) 1548 mask |= EPOLLERR; 1549 if ((sk->sk_shutdown == SHUTDOWN_MASK) || 1550 (sk->sk_state == SMC_CLOSED)) 1551 mask |= EPOLLHUP; 1552 if (sk->sk_state == SMC_LISTEN) { 1553 /* woken up by sk_data_ready in smc_listen_work() */ 1554 mask = smc_accept_poll(sk); 1555 } else { 1556 if (atomic_read(&smc->conn.sndbuf_space) || 1557 sk->sk_shutdown & SEND_SHUTDOWN) { 1558 mask |= EPOLLOUT | EPOLLWRNORM; 1559 } else { 1560 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 1561 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1562 } 1563 if (atomic_read(&smc->conn.bytes_to_rcv)) 1564 mask |= EPOLLIN | EPOLLRDNORM; 1565 if (sk->sk_shutdown & RCV_SHUTDOWN) 1566 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 1567 if (sk->sk_state == SMC_APPCLOSEWAIT1) 1568 mask |= EPOLLIN; 1569 if (smc->conn.urg_state == SMC_URG_VALID) 1570 mask |= EPOLLPRI; 1571 } 1572 } 1573 1574 return mask; 1575 } 1576 1577 static int smc_shutdown(struct socket *sock, int how) 1578 { 1579 struct sock *sk = sock->sk; 1580 struct smc_sock *smc; 1581 int rc = -EINVAL; 1582 int rc1 = 0; 1583 1584 smc = smc_sk(sk); 1585 1586 if ((how < SHUT_RD) || (how > SHUT_RDWR)) 1587 return rc; 1588 1589 lock_sock(sk); 1590 1591 rc = -ENOTCONN; 1592 if ((sk->sk_state != SMC_ACTIVE) && 1593 (sk->sk_state != SMC_PEERCLOSEWAIT1) && 1594 (sk->sk_state != SMC_PEERCLOSEWAIT2) && 1595 (sk->sk_state != SMC_APPCLOSEWAIT1) && 1596 (sk->sk_state != SMC_APPCLOSEWAIT2) && 1597 (sk->sk_state != SMC_APPFINCLOSEWAIT)) 1598 goto out; 1599 if (smc->use_fallback) { 1600 rc = kernel_sock_shutdown(smc->clcsock, how); 1601 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown; 1602 if (sk->sk_shutdown == SHUTDOWN_MASK) 1603 sk->sk_state = SMC_CLOSED; 1604 goto out; 1605 } 1606 switch (how) { 1607 case SHUT_RDWR: /* shutdown in both directions */ 1608 rc = smc_close_active(smc); 1609 break; 1610 case SHUT_WR: 1611 rc = smc_close_shutdown_write(smc); 1612 break; 1613 case SHUT_RD: 1614 rc = 0; 1615 /* nothing more to do because peer is not involved */ 1616 break; 1617 } 1618 if (smc->clcsock) 1619 rc1 = kernel_sock_shutdown(smc->clcsock, how); 1620 /* map sock_shutdown_cmd constants to sk_shutdown value range */ 1621 sk->sk_shutdown |= how + 1; 1622 1623 out: 1624 release_sock(sk); 1625 return rc ? rc : rc1; 1626 } 1627 1628 static int smc_setsockopt(struct socket *sock, int level, int optname, 1629 char __user *optval, unsigned int optlen) 1630 { 1631 struct sock *sk = sock->sk; 1632 struct smc_sock *smc; 1633 int val, rc; 1634 1635 smc = smc_sk(sk); 1636 1637 /* generic setsockopts reaching us here always apply to the 1638 * CLC socket 1639 */ 1640 rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname, 1641 optval, optlen); 1642 if (smc->clcsock->sk->sk_err) { 1643 sk->sk_err = smc->clcsock->sk->sk_err; 1644 sk->sk_error_report(sk); 1645 } 1646 if (rc) 1647 return rc; 1648 1649 if (optlen < sizeof(int)) 1650 return -EINVAL; 1651 if (get_user(val, (int __user *)optval)) 1652 return -EFAULT; 1653 1654 lock_sock(sk); 1655 switch (optname) { 1656 case TCP_ULP: 1657 case TCP_FASTOPEN: 1658 case TCP_FASTOPEN_CONNECT: 1659 case TCP_FASTOPEN_KEY: 1660 case TCP_FASTOPEN_NO_COOKIE: 1661 /* option not supported by SMC */ 1662 if (sk->sk_state == SMC_INIT) { 1663 smc->use_fallback = true; 1664 smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP; 1665 } else { 1666 if (!smc->use_fallback) 1667 rc = -EINVAL; 1668 } 1669 break; 1670 case TCP_NODELAY: 1671 if (sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) { 1672 if (val && !smc->use_fallback) 1673 mod_delayed_work(system_wq, &smc->conn.tx_work, 1674 0); 1675 } 1676 break; 1677 case TCP_CORK: 1678 if (sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) { 1679 if (!val && !smc->use_fallback) 1680 mod_delayed_work(system_wq, &smc->conn.tx_work, 1681 0); 1682 } 1683 break; 1684 case TCP_DEFER_ACCEPT: 1685 smc->sockopt_defer_accept = val; 1686 break; 1687 default: 1688 break; 1689 } 1690 release_sock(sk); 1691 1692 return rc; 1693 } 1694 1695 static int smc_getsockopt(struct socket *sock, int level, int optname, 1696 char __user *optval, int __user *optlen) 1697 { 1698 struct smc_sock *smc; 1699 1700 smc = smc_sk(sock->sk); 1701 /* socket options apply to the CLC socket */ 1702 return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname, 1703 optval, optlen); 1704 } 1705 1706 static int smc_ioctl(struct socket *sock, unsigned int cmd, 1707 unsigned long arg) 1708 { 1709 union smc_host_cursor cons, urg; 1710 struct smc_connection *conn; 1711 struct smc_sock *smc; 1712 int answ; 1713 1714 smc = smc_sk(sock->sk); 1715 conn = &smc->conn; 1716 lock_sock(&smc->sk); 1717 if (smc->use_fallback) { 1718 if (!smc->clcsock) { 1719 release_sock(&smc->sk); 1720 return -EBADF; 1721 } 1722 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg); 1723 release_sock(&smc->sk); 1724 return answ; 1725 } 1726 switch (cmd) { 1727 case SIOCINQ: /* same as FIONREAD */ 1728 if (smc->sk.sk_state == SMC_LISTEN) { 1729 release_sock(&smc->sk); 1730 return -EINVAL; 1731 } 1732 if (smc->sk.sk_state == SMC_INIT || 1733 smc->sk.sk_state == SMC_CLOSED) 1734 answ = 0; 1735 else 1736 answ = atomic_read(&smc->conn.bytes_to_rcv); 1737 break; 1738 case SIOCOUTQ: 1739 /* output queue size (not send + not acked) */ 1740 if (smc->sk.sk_state == SMC_LISTEN) { 1741 release_sock(&smc->sk); 1742 return -EINVAL; 1743 } 1744 if (smc->sk.sk_state == SMC_INIT || 1745 smc->sk.sk_state == SMC_CLOSED) 1746 answ = 0; 1747 else 1748 answ = smc->conn.sndbuf_desc->len - 1749 atomic_read(&smc->conn.sndbuf_space); 1750 break; 1751 case SIOCOUTQNSD: 1752 /* output queue size (not send only) */ 1753 if (smc->sk.sk_state == SMC_LISTEN) { 1754 release_sock(&smc->sk); 1755 return -EINVAL; 1756 } 1757 if (smc->sk.sk_state == SMC_INIT || 1758 smc->sk.sk_state == SMC_CLOSED) 1759 answ = 0; 1760 else 1761 answ = smc_tx_prepared_sends(&smc->conn); 1762 break; 1763 case SIOCATMARK: 1764 if (smc->sk.sk_state == SMC_LISTEN) { 1765 release_sock(&smc->sk); 1766 return -EINVAL; 1767 } 1768 if (smc->sk.sk_state == SMC_INIT || 1769 smc->sk.sk_state == SMC_CLOSED) { 1770 answ = 0; 1771 } else { 1772 smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn); 1773 smc_curs_copy(&urg, &conn->urg_curs, conn); 1774 answ = smc_curs_diff(conn->rmb_desc->len, 1775 &cons, &urg) == 1; 1776 } 1777 break; 1778 default: 1779 release_sock(&smc->sk); 1780 return -ENOIOCTLCMD; 1781 } 1782 release_sock(&smc->sk); 1783 1784 return put_user(answ, (int __user *)arg); 1785 } 1786 1787 static ssize_t smc_sendpage(struct socket *sock, struct page *page, 1788 int offset, size_t size, int flags) 1789 { 1790 struct sock *sk = sock->sk; 1791 struct smc_sock *smc; 1792 int rc = -EPIPE; 1793 1794 smc = smc_sk(sk); 1795 lock_sock(sk); 1796 if (sk->sk_state != SMC_ACTIVE) { 1797 release_sock(sk); 1798 goto out; 1799 } 1800 release_sock(sk); 1801 if (smc->use_fallback) 1802 rc = kernel_sendpage(smc->clcsock, page, offset, 1803 size, flags); 1804 else 1805 rc = sock_no_sendpage(sock, page, offset, size, flags); 1806 1807 out: 1808 return rc; 1809 } 1810 1811 /* Map the affected portions of the rmbe into an spd, note the number of bytes 1812 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor 1813 * updates till whenever a respective page has been fully processed. 1814 * Note that subsequent recv() calls have to wait till all splice() processing 1815 * completed. 1816 */ 1817 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos, 1818 struct pipe_inode_info *pipe, size_t len, 1819 unsigned int flags) 1820 { 1821 struct sock *sk = sock->sk; 1822 struct smc_sock *smc; 1823 int rc = -ENOTCONN; 1824 1825 smc = smc_sk(sk); 1826 lock_sock(sk); 1827 1828 if (sk->sk_state == SMC_INIT || 1829 sk->sk_state == SMC_LISTEN || 1830 sk->sk_state == SMC_CLOSED) 1831 goto out; 1832 1833 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) { 1834 rc = 0; 1835 goto out; 1836 } 1837 1838 if (smc->use_fallback) { 1839 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos, 1840 pipe, len, flags); 1841 } else { 1842 if (*ppos) { 1843 rc = -ESPIPE; 1844 goto out; 1845 } 1846 if (flags & SPLICE_F_NONBLOCK) 1847 flags = MSG_DONTWAIT; 1848 else 1849 flags = 0; 1850 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags); 1851 } 1852 out: 1853 release_sock(sk); 1854 1855 return rc; 1856 } 1857 1858 /* must look like tcp */ 1859 static const struct proto_ops smc_sock_ops = { 1860 .family = PF_SMC, 1861 .owner = THIS_MODULE, 1862 .release = smc_release, 1863 .bind = smc_bind, 1864 .connect = smc_connect, 1865 .socketpair = sock_no_socketpair, 1866 .accept = smc_accept, 1867 .getname = smc_getname, 1868 .poll = smc_poll, 1869 .ioctl = smc_ioctl, 1870 .listen = smc_listen, 1871 .shutdown = smc_shutdown, 1872 .setsockopt = smc_setsockopt, 1873 .getsockopt = smc_getsockopt, 1874 .sendmsg = smc_sendmsg, 1875 .recvmsg = smc_recvmsg, 1876 .mmap = sock_no_mmap, 1877 .sendpage = smc_sendpage, 1878 .splice_read = smc_splice_read, 1879 }; 1880 1881 static int smc_create(struct net *net, struct socket *sock, int protocol, 1882 int kern) 1883 { 1884 int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET; 1885 struct smc_sock *smc; 1886 struct sock *sk; 1887 int rc; 1888 1889 rc = -ESOCKTNOSUPPORT; 1890 if (sock->type != SOCK_STREAM) 1891 goto out; 1892 1893 rc = -EPROTONOSUPPORT; 1894 if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6) 1895 goto out; 1896 1897 rc = -ENOBUFS; 1898 sock->ops = &smc_sock_ops; 1899 sk = smc_sock_alloc(net, sock, protocol); 1900 if (!sk) 1901 goto out; 1902 1903 /* create internal TCP socket for CLC handshake and fallback */ 1904 smc = smc_sk(sk); 1905 smc->use_fallback = false; /* assume rdma capability first */ 1906 smc->fallback_rsn = 0; 1907 rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP, 1908 &smc->clcsock); 1909 if (rc) { 1910 sk_common_release(sk); 1911 goto out; 1912 } 1913 smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE); 1914 smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE); 1915 1916 out: 1917 return rc; 1918 } 1919 1920 static const struct net_proto_family smc_sock_family_ops = { 1921 .family = PF_SMC, 1922 .owner = THIS_MODULE, 1923 .create = smc_create, 1924 }; 1925 1926 static int __init smc_init(void) 1927 { 1928 int rc; 1929 1930 rc = smc_pnet_init(); 1931 if (rc) 1932 return rc; 1933 1934 rc = smc_llc_init(); 1935 if (rc) { 1936 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc); 1937 goto out_pnet; 1938 } 1939 1940 rc = smc_cdc_init(); 1941 if (rc) { 1942 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc); 1943 goto out_pnet; 1944 } 1945 1946 rc = proto_register(&smc_proto, 1); 1947 if (rc) { 1948 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc); 1949 goto out_pnet; 1950 } 1951 1952 rc = proto_register(&smc_proto6, 1); 1953 if (rc) { 1954 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc); 1955 goto out_proto; 1956 } 1957 1958 rc = sock_register(&smc_sock_family_ops); 1959 if (rc) { 1960 pr_err("%s: sock_register fails with %d\n", __func__, rc); 1961 goto out_proto6; 1962 } 1963 INIT_HLIST_HEAD(&smc_v4_hashinfo.ht); 1964 INIT_HLIST_HEAD(&smc_v6_hashinfo.ht); 1965 1966 rc = smc_ib_register_client(); 1967 if (rc) { 1968 pr_err("%s: ib_register fails with %d\n", __func__, rc); 1969 goto out_sock; 1970 } 1971 1972 static_branch_enable(&tcp_have_smc); 1973 return 0; 1974 1975 out_sock: 1976 sock_unregister(PF_SMC); 1977 out_proto6: 1978 proto_unregister(&smc_proto6); 1979 out_proto: 1980 proto_unregister(&smc_proto); 1981 out_pnet: 1982 smc_pnet_exit(); 1983 return rc; 1984 } 1985 1986 static void __exit smc_exit(void) 1987 { 1988 smc_core_exit(); 1989 static_branch_disable(&tcp_have_smc); 1990 smc_ib_unregister_client(); 1991 sock_unregister(PF_SMC); 1992 proto_unregister(&smc_proto6); 1993 proto_unregister(&smc_proto); 1994 smc_pnet_exit(); 1995 } 1996 1997 module_init(smc_init); 1998 module_exit(smc_exit); 1999 2000 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>"); 2001 MODULE_DESCRIPTION("smc socket address family"); 2002 MODULE_LICENSE("GPL"); 2003 MODULE_ALIAS_NETPROTO(PF_SMC); 2004