transport.c (6860c981b9672324cb53b883cfda8d2ea1445ff1) transport.c (eec4844fae7c033a0c1fc1eb3b8517aeb8b6cc49)
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2/*
3 * Copyright (c) 2014-2017 Oracle. All rights reserved.
4 * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file

--- 66 unchanged lines hidden (view full) ---

75int xprt_rdma_pad_optimize;
76
77#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
78
79static unsigned int min_slot_table_size = RPCRDMA_MIN_SLOT_TABLE;
80static unsigned int max_slot_table_size = RPCRDMA_MAX_SLOT_TABLE;
81static unsigned int min_inline_size = RPCRDMA_MIN_INLINE;
82static unsigned int max_inline_size = RPCRDMA_MAX_INLINE;
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2/*
3 * Copyright (c) 2014-2017 Oracle. All rights reserved.
4 * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file

--- 66 unchanged lines hidden (view full) ---

75int xprt_rdma_pad_optimize;
76
77#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
78
79static unsigned int min_slot_table_size = RPCRDMA_MIN_SLOT_TABLE;
80static unsigned int max_slot_table_size = RPCRDMA_MAX_SLOT_TABLE;
81static unsigned int min_inline_size = RPCRDMA_MIN_INLINE;
82static unsigned int max_inline_size = RPCRDMA_MAX_INLINE;
83static unsigned int zero;
84static unsigned int max_padding = PAGE_SIZE;
85static unsigned int min_memreg = RPCRDMA_BOUNCEBUFFERS;
86static unsigned int max_memreg = RPCRDMA_LAST - 1;
87static unsigned int dummy;
88
89static struct ctl_table_header *sunrpc_table_header;
90
91static struct ctl_table xr_tunables_table[] = {

--- 25 unchanged lines hidden (view full) ---

117 .extra2 = &max_inline_size,
118 },
119 {
120 .procname = "rdma_inline_write_padding",
121 .data = &dummy,
122 .maxlen = sizeof(unsigned int),
123 .mode = 0644,
124 .proc_handler = proc_dointvec_minmax,
83static unsigned int max_padding = PAGE_SIZE;
84static unsigned int min_memreg = RPCRDMA_BOUNCEBUFFERS;
85static unsigned int max_memreg = RPCRDMA_LAST - 1;
86static unsigned int dummy;
87
88static struct ctl_table_header *sunrpc_table_header;
89
90static struct ctl_table xr_tunables_table[] = {

--- 25 unchanged lines hidden (view full) ---

116 .extra2 = &max_inline_size,
117 },
118 {
119 .procname = "rdma_inline_write_padding",
120 .data = &dummy,
121 .maxlen = sizeof(unsigned int),
122 .mode = 0644,
123 .proc_handler = proc_dointvec_minmax,
125 .extra1 = &zero,
124 .extra1 = SYSCTL_ZERO,
126 .extra2 = &max_padding,
127 },
128 {
129 .procname = "rdma_memreg_strategy",
130 .data = &xprt_rdma_memreg_strategy,
131 .maxlen = sizeof(unsigned int),
132 .mode = 0644,
133 .proc_handler = proc_dointvec_minmax,

--- 159 unchanged lines hidden (view full) ---

293 rpcrdma_ia_close(&r_xprt->rx_ia);
294
295 xprt_rdma_free_addresses(xprt);
296 xprt_free(xprt);
297
298 module_put(THIS_MODULE);
299}
300
125 .extra2 = &max_padding,
126 },
127 {
128 .procname = "rdma_memreg_strategy",
129 .data = &xprt_rdma_memreg_strategy,
130 .maxlen = sizeof(unsigned int),
131 .mode = 0644,
132 .proc_handler = proc_dointvec_minmax,

--- 159 unchanged lines hidden (view full) ---

292 rpcrdma_ia_close(&r_xprt->rx_ia);
293
294 xprt_rdma_free_addresses(xprt);
295 xprt_free(xprt);
296
297 module_put(THIS_MODULE);
298}
299
301/* 60 second timeout, no retries */
302static const struct rpc_timeout xprt_rdma_default_timeout = {
303 .to_initval = 60 * HZ,
304 .to_maxval = 60 * HZ,
305};
306
307/**
308 * xprt_setup_rdma - Set up transport to use RDMA
309 *

--- 9 unchanged lines hidden (view full) ---

319
320 if (args->addrlen > sizeof(xprt->addr))
321 return ERR_PTR(-EBADF);
322
323 xprt = xprt_alloc(args->net, sizeof(struct rpcrdma_xprt), 0, 0);
324 if (!xprt)
325 return ERR_PTR(-ENOMEM);
326
300static const struct rpc_timeout xprt_rdma_default_timeout = {
301 .to_initval = 60 * HZ,
302 .to_maxval = 60 * HZ,
303};
304
305/**
306 * xprt_setup_rdma - Set up transport to use RDMA
307 *

--- 9 unchanged lines hidden (view full) ---

317
318 if (args->addrlen > sizeof(xprt->addr))
319 return ERR_PTR(-EBADF);
320
321 xprt = xprt_alloc(args->net, sizeof(struct rpcrdma_xprt), 0, 0);
322 if (!xprt)
323 return ERR_PTR(-ENOMEM);
324
325 /* 60 second timeout, no retries */
327 xprt->timeout = &xprt_rdma_default_timeout;
326 xprt->timeout = &xprt_rdma_default_timeout;
328 xprt->connect_timeout = xprt->timeout->to_initval;
329 xprt->max_reconnect_timeout = xprt->timeout->to_maxval;
330 xprt->bind_timeout = RPCRDMA_BIND_TO;
331 xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO;
332 xprt->idle_timeout = RPCRDMA_IDLE_DISC_TO;
333
334 xprt->resvport = 0; /* privileged port not needed */
335 xprt->ops = &xprt_rdma_procs;
336
337 /*

--- 146 unchanged lines hidden (view full) ---

484 */
485static void
486xprt_rdma_timer(struct rpc_xprt *xprt, struct rpc_task *task)
487{
488 xprt_force_disconnect(xprt);
489}
490
491/**
327 xprt->bind_timeout = RPCRDMA_BIND_TO;
328 xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO;
329 xprt->idle_timeout = RPCRDMA_IDLE_DISC_TO;
330
331 xprt->resvport = 0; /* privileged port not needed */
332 xprt->ops = &xprt_rdma_procs;
333
334 /*

--- 146 unchanged lines hidden (view full) ---

481 */
482static void
483xprt_rdma_timer(struct rpc_xprt *xprt, struct rpc_task *task)
484{
485 xprt_force_disconnect(xprt);
486}
487
488/**
492 * xprt_rdma_set_connect_timeout - set timeouts for establishing a connection
493 * @xprt: controlling transport instance
494 * @connect_timeout: reconnect timeout after client disconnects
495 * @reconnect_timeout: reconnect timeout after server disconnects
496 *
497 */
498static void xprt_rdma_tcp_set_connect_timeout(struct rpc_xprt *xprt,
499 unsigned long connect_timeout,
500 unsigned long reconnect_timeout)
501{
502 struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
503
504 trace_xprtrdma_op_set_cto(r_xprt, connect_timeout, reconnect_timeout);
505
506 spin_lock(&xprt->transport_lock);
507
508 if (connect_timeout < xprt->connect_timeout) {
509 struct rpc_timeout to;
510 unsigned long initval;
511
512 to = *xprt->timeout;
513 initval = connect_timeout;
514 if (initval < RPCRDMA_INIT_REEST_TO << 1)
515 initval = RPCRDMA_INIT_REEST_TO << 1;
516 to.to_initval = initval;
517 to.to_maxval = initval;
518 r_xprt->rx_timeout = to;
519 xprt->timeout = &r_xprt->rx_timeout;
520 xprt->connect_timeout = connect_timeout;
521 }
522
523 if (reconnect_timeout < xprt->max_reconnect_timeout)
524 xprt->max_reconnect_timeout = reconnect_timeout;
525
526 spin_unlock(&xprt->transport_lock);
527}
528
529/**
530 * xprt_rdma_connect - schedule an attempt to reconnect
489 * xprt_rdma_connect - try to establish a transport connection
531 * @xprt: transport state
490 * @xprt: transport state
532 * @task: RPC scheduler context (unused)
491 * @task: RPC scheduler context
533 *
534 */
535static void
536xprt_rdma_connect(struct rpc_xprt *xprt, struct rpc_task *task)
537{
538 struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
492 *
493 */
494static void
495xprt_rdma_connect(struct rpc_xprt *xprt, struct rpc_task *task)
496{
497 struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
539 unsigned long delay;
540
541 trace_xprtrdma_op_connect(r_xprt);
498
499 trace_xprtrdma_op_connect(r_xprt);
542
543 delay = 0;
544 if (r_xprt->rx_ep.rep_connected != 0) {
500 if (r_xprt->rx_ep.rep_connected != 0) {
545 delay = xprt_reconnect_delay(xprt);
546 xprt_reconnect_backoff(xprt, RPCRDMA_INIT_REEST_TO);
501 /* Reconnect */
502 schedule_delayed_work(&r_xprt->rx_connect_worker,
503 xprt->reestablish_timeout);
504 xprt->reestablish_timeout <<= 1;
505 if (xprt->reestablish_timeout > RPCRDMA_MAX_REEST_TO)
506 xprt->reestablish_timeout = RPCRDMA_MAX_REEST_TO;
507 else if (xprt->reestablish_timeout < RPCRDMA_INIT_REEST_TO)
508 xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO;
509 } else {
510 schedule_delayed_work(&r_xprt->rx_connect_worker, 0);
511 if (!RPC_IS_ASYNC(task))
512 flush_delayed_work(&r_xprt->rx_connect_worker);
547 }
513 }
548 queue_delayed_work(xprtiod_workqueue, &r_xprt->rx_connect_worker,
549 delay);
550}
551
552/**
553 * xprt_rdma_alloc_slot - allocate an rpc_rqst
554 * @xprt: controlling RPC transport
555 * @task: RPC task requesting a fresh rpc_rqst
556 *
557 * tk_status values:

--- 22 unchanged lines hidden (view full) ---

580 * xprt_rdma_free_slot - release an rpc_rqst
581 * @xprt: controlling RPC transport
582 * @rqst: rpc_rqst to release
583 *
584 */
585static void
586xprt_rdma_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *rqst)
587{
514}
515
516/**
517 * xprt_rdma_alloc_slot - allocate an rpc_rqst
518 * @xprt: controlling RPC transport
519 * @task: RPC task requesting a fresh rpc_rqst
520 *
521 * tk_status values:

--- 22 unchanged lines hidden (view full) ---

544 * xprt_rdma_free_slot - release an rpc_rqst
545 * @xprt: controlling RPC transport
546 * @rqst: rpc_rqst to release
547 *
548 */
549static void
550xprt_rdma_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *rqst)
551{
588 struct rpcrdma_xprt *r_xprt =
589 container_of(xprt, struct rpcrdma_xprt, rx_xprt);
590
591 memset(rqst, 0, sizeof(*rqst));
552 memset(rqst, 0, sizeof(*rqst));
592 rpcrdma_buffer_put(&r_xprt->rx_buf, rpcr_to_rdmar(rqst));
553 rpcrdma_buffer_put(rpcr_to_rdmar(rqst));
593 rpc_wake_up_next(&xprt->backlog);
594}
595
596static bool rpcrdma_check_regbuf(struct rpcrdma_xprt *r_xprt,
597 struct rpcrdma_regbuf *rb, size_t size,
598 gfp_t flags)
599{
600 if (unlikely(rdmab_length(rb) < size)) {

--- 50 unchanged lines hidden (view full) ---

651 */
652static void
653xprt_rdma_free(struct rpc_task *task)
654{
655 struct rpc_rqst *rqst = task->tk_rqstp;
656 struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt);
657 struct rpcrdma_req *req = rpcr_to_rdmar(rqst);
658
554 rpc_wake_up_next(&xprt->backlog);
555}
556
557static bool rpcrdma_check_regbuf(struct rpcrdma_xprt *r_xprt,
558 struct rpcrdma_regbuf *rb, size_t size,
559 gfp_t flags)
560{
561 if (unlikely(rdmab_length(rb) < size)) {

--- 50 unchanged lines hidden (view full) ---

612 */
613static void
614xprt_rdma_free(struct rpc_task *task)
615{
616 struct rpc_rqst *rqst = task->tk_rqstp;
617 struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt);
618 struct rpcrdma_req *req = rpcr_to_rdmar(rqst);
619
620 if (test_bit(RPCRDMA_REQ_F_PENDING, &req->rl_flags))
621 rpcrdma_release_rqst(r_xprt, req);
659 trace_xprtrdma_op_free(task, req);
622 trace_xprtrdma_op_free(task, req);
660
661 if (!list_empty(&req->rl_registered))
662 frwr_unmap_sync(r_xprt, req);
663
664 /* XXX: If the RPC is completing because of a signal and
665 * not because a reply was received, we ought to ensure
666 * that the Send completion has fired, so that memory
667 * involved with the Send is not still visible to the NIC.
668 */
669}
670
671/**
672 * xprt_rdma_send_request - marshal and send an RPC request
673 * @rqst: RPC message in rq_snd_buf
674 *
675 * Caller holds the transport's write lock.
676 *

--- 30 unchanged lines hidden (view full) ---

707 if (rc < 0)
708 goto failed_marshal;
709
710 /* Must suppress retransmit to maintain credits */
711 if (rqst->rq_connect_cookie == xprt->connect_cookie)
712 goto drop_connection;
713 rqst->rq_xtime = ktime_get();
714
623}
624
625/**
626 * xprt_rdma_send_request - marshal and send an RPC request
627 * @rqst: RPC message in rq_snd_buf
628 *
629 * Caller holds the transport's write lock.
630 *

--- 30 unchanged lines hidden (view full) ---

661 if (rc < 0)
662 goto failed_marshal;
663
664 /* Must suppress retransmit to maintain credits */
665 if (rqst->rq_connect_cookie == xprt->connect_cookie)
666 goto drop_connection;
667 rqst->rq_xtime = ktime_get();
668
669 __set_bit(RPCRDMA_REQ_F_PENDING, &req->rl_flags);
715 if (rpcrdma_ep_post(&r_xprt->rx_ia, &r_xprt->rx_ep, req))
716 goto drop_connection;
717
718 rqst->rq_xmit_bytes_sent += rqst->rq_snd_buf.len;
719
720 /* An RPC with no reply will throw off credit accounting,
721 * so drop the connection to reset the credit grant.
722 */

--- 76 unchanged lines hidden (view full) ---

799 .rpcbind = rpcb_getport_async, /* sunrpc/rpcb_clnt.c */
800 .set_port = xprt_rdma_set_port,
801 .connect = xprt_rdma_connect,
802 .buf_alloc = xprt_rdma_allocate,
803 .buf_free = xprt_rdma_free,
804 .send_request = xprt_rdma_send_request,
805 .close = xprt_rdma_close,
806 .destroy = xprt_rdma_destroy,
670 if (rpcrdma_ep_post(&r_xprt->rx_ia, &r_xprt->rx_ep, req))
671 goto drop_connection;
672
673 rqst->rq_xmit_bytes_sent += rqst->rq_snd_buf.len;
674
675 /* An RPC with no reply will throw off credit accounting,
676 * so drop the connection to reset the credit grant.
677 */

--- 76 unchanged lines hidden (view full) ---

754 .rpcbind = rpcb_getport_async, /* sunrpc/rpcb_clnt.c */
755 .set_port = xprt_rdma_set_port,
756 .connect = xprt_rdma_connect,
757 .buf_alloc = xprt_rdma_allocate,
758 .buf_free = xprt_rdma_free,
759 .send_request = xprt_rdma_send_request,
760 .close = xprt_rdma_close,
761 .destroy = xprt_rdma_destroy,
807 .set_connect_timeout = xprt_rdma_tcp_set_connect_timeout,
808 .print_stats = xprt_rdma_print_stats,
809 .enable_swap = xprt_rdma_enable_swap,
810 .disable_swap = xprt_rdma_disable_swap,
811 .inject_disconnect = xprt_rdma_inject_disconnect,
812#if defined(CONFIG_SUNRPC_BACKCHANNEL)
813 .bc_setup = xprt_rdma_bc_setup,
814 .bc_maxpayload = xprt_rdma_bc_maxpayload,
762 .print_stats = xprt_rdma_print_stats,
763 .enable_swap = xprt_rdma_enable_swap,
764 .disable_swap = xprt_rdma_disable_swap,
765 .inject_disconnect = xprt_rdma_inject_disconnect,
766#if defined(CONFIG_SUNRPC_BACKCHANNEL)
767 .bc_setup = xprt_rdma_bc_setup,
768 .bc_maxpayload = xprt_rdma_bc_maxpayload,
815 .bc_num_slots = xprt_rdma_bc_max_slots,
816 .bc_free_rqst = xprt_rdma_bc_free_rqst,
817 .bc_destroy = xprt_rdma_bc_destroy,
818#endif
819};
820
821static struct xprt_class xprt_rdma = {
822 .list = LIST_HEAD_INIT(xprt_rdma.list),
823 .name = "rdma",

--- 38 unchanged lines hidden --- 		769 .bc_free_rqst = xprt_rdma_bc_free_rqst,
770 .bc_destroy = xprt_rdma_bc_destroy,
771#endif
772};
773
774static struct xprt_class xprt_rdma = {
775 .list = LIST_HEAD_INIT(xprt_rdma.list),
776 .name = "rdma",

--- 38 unchanged lines hidden ---