connection.c (73ab603f44149ff48889d5109ea9ab64bf38cc69) connection.c (eee2fa6ab3225192d6d894c54a6fb02ac9efdff6)
1/*
1/*
2 * Copyright (c) 2006 Oracle. All rights reserved.
2 * Copyright (c) 2006, 2017 Oracle and/or its affiliates. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or

--- 18 unchanged lines hidden (view full) ---

29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33#include <linux/kernel.h>
34#include <linux/list.h>
35#include <linux/slab.h>
36#include <linux/export.h>
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or

--- 18 unchanged lines hidden (view full) ---

29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33#include <linux/kernel.h>
34#include <linux/list.h>
35#include <linux/slab.h>
36#include <linux/export.h>
37#include <net/inet_hashtables.h>
37#include <net/ipv6.h>
38#include <net/inet6_hashtables.h>
38
39#include "rds.h"
40#include "loop.h"
41
42#define RDS_CONNECTION_HASH_BITS 12
43#define RDS_CONNECTION_HASH_ENTRIES (1 << RDS_CONNECTION_HASH_BITS)
44#define RDS_CONNECTION_HASH_MASK (RDS_CONNECTION_HASH_ENTRIES - 1)
45
46/* converting this to RCU is a chore for another day.. */
47static DEFINE_SPINLOCK(rds_conn_lock);
48static unsigned long rds_conn_count;
49static struct hlist_head rds_conn_hash[RDS_CONNECTION_HASH_ENTRIES];
50static struct kmem_cache *rds_conn_slab;
51
39
40#include "rds.h"
41#include "loop.h"
42
43#define RDS_CONNECTION_HASH_BITS 12
44#define RDS_CONNECTION_HASH_ENTRIES (1 << RDS_CONNECTION_HASH_BITS)
45#define RDS_CONNECTION_HASH_MASK (RDS_CONNECTION_HASH_ENTRIES - 1)
46
47/* converting this to RCU is a chore for another day.. */
48static DEFINE_SPINLOCK(rds_conn_lock);
49static unsigned long rds_conn_count;
50static struct hlist_head rds_conn_hash[RDS_CONNECTION_HASH_ENTRIES];
51static struct kmem_cache *rds_conn_slab;
52
52static struct hlist_head *rds_conn_bucket(__be32 laddr, __be32 faddr)
53static struct hlist_head *rds_conn_bucket(const struct in6_addr *laddr,
54 const struct in6_addr *faddr)
53{
55{
56 static u32 rds6_hash_secret __read_mostly;
54 static u32 rds_hash_secret __read_mostly;
55
57 static u32 rds_hash_secret __read_mostly;
58
56 unsigned long hash;
59 u32 lhash, fhash, hash;
57
58 net_get_random_once(&rds_hash_secret, sizeof(rds_hash_secret));
60
61 net_get_random_once(&rds_hash_secret, sizeof(rds_hash_secret));
62 net_get_random_once(&rds6_hash_secret, sizeof(rds6_hash_secret));
59
63
60 /* Pass NULL, don't need struct net for hash */
61 hash = __inet_ehashfn(be32_to_cpu(laddr), 0,
62 be32_to_cpu(faddr), 0,
63 rds_hash_secret);
64 lhash = (__force u32)laddr->s6_addr32[3];
65 fhash = __ipv6_addr_jhash(faddr, rds6_hash_secret);
66 hash = __inet6_ehashfn(lhash, 0, fhash, 0, rds_hash_secret);
67
64 return &rds_conn_hash[hash & RDS_CONNECTION_HASH_MASK];
65}
66
67#define rds_conn_info_set(var, test, suffix) do { \
68 if (test) \
69 var |= RDS_INFO_CONNECTION_FLAG_##suffix; \
70} while (0)
71
72/* rcu read lock must be held or the connection spinlock */
73static struct rds_connection *rds_conn_lookup(struct net *net,
74 struct hlist_head *head,
68 return &rds_conn_hash[hash & RDS_CONNECTION_HASH_MASK];
69}
70
71#define rds_conn_info_set(var, test, suffix) do { \
72 if (test) \
73 var |= RDS_INFO_CONNECTION_FLAG_##suffix; \
74} while (0)
75
76/* rcu read lock must be held or the connection spinlock */
77static struct rds_connection *rds_conn_lookup(struct net *net,
78 struct hlist_head *head,
75 __be32 laddr, __be32 faddr,
76 struct rds_transport *trans)
79 const struct in6_addr *laddr,
80 const struct in6_addr *faddr,
81 struct rds_transport *trans,
82 int dev_if)
77{
78 struct rds_connection *conn, *ret = NULL;
79
80 hlist_for_each_entry_rcu(conn, head, c_hash_node) {
83{
84 struct rds_connection *conn, *ret = NULL;
85
86 hlist_for_each_entry_rcu(conn, head, c_hash_node) {
81 if (conn->c_faddr == faddr && conn->c_laddr == laddr &&
82 conn->c_trans == trans && net == rds_conn_net(conn)) {
87 if (ipv6_addr_equal(&conn->c_faddr, faddr) &&
88 ipv6_addr_equal(&conn->c_laddr, laddr) &&
89 conn->c_trans == trans &&
90 net == rds_conn_net(conn) &&
91 conn->c_dev_if == dev_if) {
83 ret = conn;
84 break;
85 }
86 }
92 ret = conn;
93 break;
94 }
95 }
87 rdsdebug("returning conn %p for %pI4 -> %pI4\n", ret,
88 &laddr, &faddr);
96 rdsdebug("returning conn %p for %pI6c -> %pI6c\n", ret,
97 laddr, faddr);
89 return ret;
90}
91
92/*
93 * This is called by transports as they're bringing down a connection.
94 * It clears partial message state so that the transport can start sending
95 * and receiving over this connection again in the future. It is up to
96 * the transport to have serialized this call with its send and recv.
97 */
98static void rds_conn_path_reset(struct rds_conn_path *cp)
99{
100 struct rds_connection *conn = cp->cp_conn;
101
98 return ret;
99}
100
101/*
102 * This is called by transports as they're bringing down a connection.
103 * It clears partial message state so that the transport can start sending
104 * and receiving over this connection again in the future. It is up to
105 * the transport to have serialized this call with its send and recv.
106 */
107static void rds_conn_path_reset(struct rds_conn_path *cp)
108{
109 struct rds_connection *conn = cp->cp_conn;
110
102 rdsdebug("connection %pI4 to %pI4 reset\n",
103 &conn->c_laddr, &conn->c_faddr);
111 rdsdebug("connection %pI6c to %pI6c reset\n",
112 &conn->c_laddr, &conn->c_faddr);
104
105 rds_stats_inc(s_conn_reset);
106 rds_send_path_reset(cp);
107 cp->cp_flags = 0;
108
109 /* Do not clear next_rx_seq here, else we cannot distinguish
110 * retransmitted packets from new packets, and will hand all
111 * of them to the application. That is not consistent with the

--- 25 unchanged lines hidden (view full) ---

137 * There is only every one 'conn' for a given pair of addresses in the
138 * system at a time. They contain messages to be retransmitted and so
139 * span the lifetime of the actual underlying transport connections.
140 *
141 * For now they are not garbage collected once they're created. They
142 * are torn down as the module is removed, if ever.
143 */
144static struct rds_connection *__rds_conn_create(struct net *net,
113
114 rds_stats_inc(s_conn_reset);
115 rds_send_path_reset(cp);
116 cp->cp_flags = 0;
117
118 /* Do not clear next_rx_seq here, else we cannot distinguish
119 * retransmitted packets from new packets, and will hand all
120 * of them to the application. That is not consistent with the

--- 25 unchanged lines hidden (view full) ---

146 * There is only every one 'conn' for a given pair of addresses in the
147 * system at a time. They contain messages to be retransmitted and so
148 * span the lifetime of the actual underlying transport connections.
149 *
150 * For now they are not garbage collected once they're created. They
151 * are torn down as the module is removed, if ever.
152 */
153static struct rds_connection *__rds_conn_create(struct net *net,
145 __be32 laddr, __be32 faddr,
146 struct rds_transport *trans, gfp_t gfp,
147 int is_outgoing)
154 const struct in6_addr *laddr,
155 const struct in6_addr *faddr,
156 struct rds_transport *trans,
157 gfp_t gfp,
158 int is_outgoing,
159 int dev_if)
148{
149 struct rds_connection *conn, *parent = NULL;
150 struct hlist_head *head = rds_conn_bucket(laddr, faddr);
151 struct rds_transport *loop_trans;
152 unsigned long flags;
153 int ret, i;
154 int npaths = (trans->t_mp_capable ? RDS_MPATH_WORKERS : 1);
155
156 rcu_read_lock();
160{
161 struct rds_connection *conn, *parent = NULL;
162 struct hlist_head *head = rds_conn_bucket(laddr, faddr);
163 struct rds_transport *loop_trans;
164 unsigned long flags;
165 int ret, i;
166 int npaths = (trans->t_mp_capable ? RDS_MPATH_WORKERS : 1);
167
168 rcu_read_lock();
157 conn = rds_conn_lookup(net, head, laddr, faddr, trans);
158 if (conn && conn->c_loopback && conn->c_trans != &rds_loop_transport &&
159 laddr == faddr && !is_outgoing) {
169 conn = rds_conn_lookup(net, head, laddr, faddr, trans, dev_if);
170 if (conn &&
171 conn->c_loopback &&
172 conn->c_trans != &rds_loop_transport &&
173 ipv6_addr_equal(laddr, faddr) &&
174 !is_outgoing) {
160 /* This is a looped back IB connection, and we're
161 * called by the code handling the incoming connect.
162 * We need a second connection object into which we
163 * can stick the other QP. */
164 parent = conn;
165 conn = parent->c_passive;
166 }
167 rcu_read_unlock();

--- 8 unchanged lines hidden (view full) ---

176 conn->c_path = kcalloc(npaths, sizeof(struct rds_conn_path), gfp);
177 if (!conn->c_path) {
178 kmem_cache_free(rds_conn_slab, conn);
179 conn = ERR_PTR(-ENOMEM);
180 goto out;
181 }
182
183 INIT_HLIST_NODE(&conn->c_hash_node);
175 /* This is a looped back IB connection, and we're
176 * called by the code handling the incoming connect.
177 * We need a second connection object into which we
178 * can stick the other QP. */
179 parent = conn;
180 conn = parent->c_passive;
181 }
182 rcu_read_unlock();

--- 8 unchanged lines hidden (view full) ---

191 conn->c_path = kcalloc(npaths, sizeof(struct rds_conn_path), gfp);
192 if (!conn->c_path) {
193 kmem_cache_free(rds_conn_slab, conn);
194 conn = ERR_PTR(-ENOMEM);
195 goto out;
196 }
197
198 INIT_HLIST_NODE(&conn->c_hash_node);
184 conn->c_laddr = laddr;
185 conn->c_faddr = faddr;
199 conn->c_laddr = *laddr;
200 conn->c_isv6 = !ipv6_addr_v4mapped(laddr);
201 conn->c_faddr = *faddr;
202 conn->c_dev_if = dev_if;
186
187 rds_conn_net_set(conn, net);
188
189 ret = rds_cong_get_maps(conn);
190 if (ret) {
191 kfree(conn->c_path);
192 kmem_cache_free(rds_conn_slab, conn);
193 conn = ERR_PTR(ret);
194 goto out;
195 }
196
197 /*
198 * This is where a connection becomes loopback. If *any* RDS sockets
199 * can bind to the destination address then we'd rather the messages
200 * flow through loopback rather than either transport.
201 */
203
204 rds_conn_net_set(conn, net);
205
206 ret = rds_cong_get_maps(conn);
207 if (ret) {
208 kfree(conn->c_path);
209 kmem_cache_free(rds_conn_slab, conn);
210 conn = ERR_PTR(ret);
211 goto out;
212 }
213
214 /*
215 * This is where a connection becomes loopback. If *any* RDS sockets
216 * can bind to the destination address then we'd rather the messages
217 * flow through loopback rather than either transport.
218 */
202 loop_trans = rds_trans_get_preferred(net, faddr);
219 loop_trans = rds_trans_get_preferred(net, faddr, conn->c_dev_if);
203 if (loop_trans) {
204 rds_trans_put(loop_trans);
205 conn->c_loopback = 1;
206 if (is_outgoing && trans->t_prefer_loopback) {
207 /* "outgoing" connection - and the transport
208 * says it wants the connection handled by the
209 * loopback transport. This is what TCP does.
210 */

--- 17 unchanged lines hidden (view full) ---

228 if (ret) {
229 rcu_read_unlock();
230 kfree(conn->c_path);
231 kmem_cache_free(rds_conn_slab, conn);
232 conn = ERR_PTR(ret);
233 goto out;
234 }
235
220 if (loop_trans) {
221 rds_trans_put(loop_trans);
222 conn->c_loopback = 1;
223 if (is_outgoing && trans->t_prefer_loopback) {
224 /* "outgoing" connection - and the transport
225 * says it wants the connection handled by the
226 * loopback transport. This is what TCP does.
227 */

--- 17 unchanged lines hidden (view full) ---

245 if (ret) {
246 rcu_read_unlock();
247 kfree(conn->c_path);
248 kmem_cache_free(rds_conn_slab, conn);
249 conn = ERR_PTR(ret);
250 goto out;
251 }
252
236 rdsdebug("allocated conn %p for %pI4 -> %pI4 over %s %s\n",
237 conn, &laddr, &faddr,
238 strnlen(trans->t_name, sizeof(trans->t_name)) ? trans->t_name :
239 "[unknown]", is_outgoing ? "(outgoing)" : "");
253 rdsdebug("allocated conn %p for %pI6c -> %pI6c over %s %s\n",
254 conn, laddr, faddr,
255 strnlen(trans->t_name, sizeof(trans->t_name)) ?
256 trans->t_name : "[unknown]", is_outgoing ? "(outgoing)" : "");
240
241 /*
242 * Since we ran without holding the conn lock, someone could
243 * have created the same conn (either normal or passive) in the
244 * interim. We check while holding the lock. If we won, we complete
245 * init and return our conn. If we lost, we rollback and return the
246 * other one.
247 */

--- 9 unchanged lines hidden (view full) ---

257 parent->c_passive = conn;
258 rds_cong_add_conn(conn);
259 rds_conn_count++;
260 }
261 } else {
262 /* Creating normal conn */
263 struct rds_connection *found;
264
257
258 /*
259 * Since we ran without holding the conn lock, someone could
260 * have created the same conn (either normal or passive) in the
261 * interim. We check while holding the lock. If we won, we complete
262 * init and return our conn. If we lost, we rollback and return the
263 * other one.
264 */

--- 9 unchanged lines hidden (view full) ---

274 parent->c_passive = conn;
275 rds_cong_add_conn(conn);
276 rds_conn_count++;
277 }
278 } else {
279 /* Creating normal conn */
280 struct rds_connection *found;
281
265 found = rds_conn_lookup(net, head, laddr, faddr, trans);
282 found = rds_conn_lookup(net, head, laddr, faddr, trans,
283 dev_if);
266 if (found) {
267 struct rds_conn_path *cp;
268 int i;
269
270 for (i = 0; i < npaths; i++) {
271 cp = &conn->c_path[i];
272 /* The ->conn_alloc invocation may have
273 * allocated resource for all paths, so all

--- 16 unchanged lines hidden (view full) ---

290 spin_unlock_irqrestore(&rds_conn_lock, flags);
291 rcu_read_unlock();
292
293out:
294 return conn;
295}
296
297struct rds_connection *rds_conn_create(struct net *net,
284 if (found) {
285 struct rds_conn_path *cp;
286 int i;
287
288 for (i = 0; i < npaths; i++) {
289 cp = &conn->c_path[i];
290 /* The ->conn_alloc invocation may have
291 * allocated resource for all paths, so all

--- 16 unchanged lines hidden (view full) ---

308 spin_unlock_irqrestore(&rds_conn_lock, flags);
309 rcu_read_unlock();
310
311out:
312 return conn;
313}
314
315struct rds_connection *rds_conn_create(struct net *net,
298 __be32 laddr, __be32 faddr,
299 struct rds_transport *trans, gfp_t gfp)
316 const struct in6_addr *laddr,
317 const struct in6_addr *faddr,
318 struct rds_transport *trans, gfp_t gfp,
319 int dev_if)
300{
320{
301 return __rds_conn_create(net, laddr, faddr, trans, gfp, 0);
321 return __rds_conn_create(net, laddr, faddr, trans, gfp, 0, dev_if);
302}
303EXPORT_SYMBOL_GPL(rds_conn_create);
304
305struct rds_connection *rds_conn_create_outgoing(struct net *net,
322}
323EXPORT_SYMBOL_GPL(rds_conn_create);
324
325struct rds_connection *rds_conn_create_outgoing(struct net *net,
306 __be32 laddr, __be32 faddr,
307 struct rds_transport *trans, gfp_t gfp)
326 const struct in6_addr *laddr,
327 const struct in6_addr *faddr,
328 struct rds_transport *trans,
329 gfp_t gfp, int dev_if)
308{
330{
309 return __rds_conn_create(net, laddr, faddr, trans, gfp, 1);
331 return __rds_conn_create(net, laddr, faddr, trans, gfp, 1, dev_if);
310}
311EXPORT_SYMBOL_GPL(rds_conn_create_outgoing);
312
313void rds_conn_shutdown(struct rds_conn_path *cp)
314{
315 struct rds_connection *conn = cp->cp_conn;
316
317 /* shut it down unless it's down already */

--- 179 unchanged lines hidden (view full) ---

497 list = &cp->cp_send_queue;
498 else
499 list = &cp->cp_retrans;
500
501 spin_lock_irqsave(&cp->cp_lock, flags);
502
503 /* XXX too lazy to maintain counts.. */
504 list_for_each_entry(rm, list, m_conn_item) {
332}
333EXPORT_SYMBOL_GPL(rds_conn_create_outgoing);
334
335void rds_conn_shutdown(struct rds_conn_path *cp)
336{
337 struct rds_connection *conn = cp->cp_conn;
338
339 /* shut it down unless it's down already */

--- 179 unchanged lines hidden (view full) ---

519 list = &cp->cp_send_queue;
520 else
521 list = &cp->cp_retrans;
522
523 spin_lock_irqsave(&cp->cp_lock, flags);
524
525 /* XXX too lazy to maintain counts.. */
526 list_for_each_entry(rm, list, m_conn_item) {
527 __be32 laddr;
528 __be32 faddr;
529
505 total++;
530 total++;
531 laddr = conn->c_laddr.s6_addr32[3];
532 faddr = conn->c_faddr.s6_addr32[3];
506 if (total <= len)
507 rds_inc_info_copy(&rm->m_inc,
508 iter,
533 if (total <= len)
534 rds_inc_info_copy(&rm->m_inc,
535 iter,
509 conn->c_laddr,
510 conn->c_faddr,
536 laddr,
537 faddr,
511 0);
512 }
513
514 spin_unlock_irqrestore(&cp->cp_lock, flags);
515 }
516 }
517 }
518 rcu_read_unlock();

--- 60 unchanged lines hidden (view full) ---

579 struct rds_info_lengths *lens,
580 int (*visitor)(struct rds_conn_path *, void *),
581 u64 *buffer,
582 size_t item_len)
583{
584 struct hlist_head *head;
585 struct rds_connection *conn;
586 size_t i;
538 0);
539 }
540
541 spin_unlock_irqrestore(&cp->cp_lock, flags);
542 }
543 }
544 }
545 rcu_read_unlock();

--- 60 unchanged lines hidden (view full) ---

606 struct rds_info_lengths *lens,
607 int (*visitor)(struct rds_conn_path *, void *),
608 u64 *buffer,
609 size_t item_len)
610{
611 struct hlist_head *head;
612 struct rds_connection *conn;
613 size_t i;
587 int j;
588
589 rcu_read_lock();
590
591 lens->nr = 0;
592 lens->each = item_len;
593
594 for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash);
595 i++, head++) {
596 hlist_for_each_entry_rcu(conn, head, c_hash_node) {
597 struct rds_conn_path *cp;
614
615 rcu_read_lock();
616
617 lens->nr = 0;
618 lens->each = item_len;
619
620 for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash);
621 i++, head++) {
622 hlist_for_each_entry_rcu(conn, head, c_hash_node) {
623 struct rds_conn_path *cp;
598 int npaths;
599
624
600 npaths = (conn->c_trans->t_mp_capable ?
601 RDS_MPATH_WORKERS : 1);
602 for (j = 0; j < npaths; j++) {
603 cp = &conn->c_path[j];
625 /* XXX We only copy the information from the first
626 * path for now. The problem is that if there are
627 * more than one underlying paths, we cannot report
628 * information of all of them using the existing
629 * API. For example, there is only one next_tx_seq,
630 * which path's next_tx_seq should we report? It is
631 * a bug in the design of MPRDS.
632 */
633 cp = conn->c_path;
604
634
605 /* XXX no cp_lock usage.. */
606 if (!visitor(cp, buffer))
607 continue;
608 }
635 /* XXX no cp_lock usage.. */
636 if (!visitor(cp, buffer))
637 continue;
609
610 /* We copy as much as we can fit in the buffer,
611 * but we count all items so that the caller
612 * can resize the buffer.
613 */
614 if (len >= item_len) {
615 rds_info_copy(iter, buffer, item_len);
616 len -= item_len;
617 }
618 lens->nr++;
619 }
620 }
621 rcu_read_unlock();
622}
623
624static int rds_conn_info_visitor(struct rds_conn_path *cp, void *buffer)
625{
626 struct rds_info_connection *cinfo = buffer;
638
639 /* We copy as much as we can fit in the buffer,
640 * but we count all items so that the caller
641 * can resize the buffer.
642 */
643 if (len >= item_len) {
644 rds_info_copy(iter, buffer, item_len);
645 len -= item_len;
646 }
647 lens->nr++;
648 }
649 }
650 rcu_read_unlock();
651}
652
653static int rds_conn_info_visitor(struct rds_conn_path *cp, void *buffer)
654{
655 struct rds_info_connection *cinfo = buffer;
656 struct rds_connection *conn = cp->cp_conn;
627
628 cinfo->next_tx_seq = cp->cp_next_tx_seq;
629 cinfo->next_rx_seq = cp->cp_next_rx_seq;
657
658 cinfo->next_tx_seq = cp->cp_next_tx_seq;
659 cinfo->next_rx_seq = cp->cp_next_rx_seq;
630 cinfo->laddr = cp->cp_conn->c_laddr;
631 cinfo->faddr = cp->cp_conn->c_faddr;
632 strncpy(cinfo->transport, cp->cp_conn->c_trans->t_name,
660 cinfo->laddr = conn->c_laddr.s6_addr32[3];
661 cinfo->faddr = conn->c_faddr.s6_addr32[3];
662 strncpy(cinfo->transport, conn->c_trans->t_name,
633 sizeof(cinfo->transport));
634 cinfo->flags = 0;
635
636 rds_conn_info_set(cinfo->flags, test_bit(RDS_IN_XMIT, &cp->cp_flags),
637 SENDING);
638 /* XXX Future: return the state rather than these funky bits */
639 rds_conn_info_set(cinfo->flags,
640 atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING,

--- 120 unchanged lines hidden --- 		663 sizeof(cinfo->transport));
664 cinfo->flags = 0;
665
666 rds_conn_info_set(cinfo->flags, test_bit(RDS_IN_XMIT, &cp->cp_flags),
667 SENDING);
668 /* XXX Future: return the state rather than these funky bits */
669 rds_conn_info_set(cinfo->flags,
670 atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING,

--- 120 unchanged lines hidden ---