1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2
3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4 /* Kai Shen <kaishen@linux.alibaba.com> */
5 /* Copyright (c) 2020-2022, Alibaba Group. */
6
7 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
8 /* Fredy Neeser */
9 /* Greg Joyce <greg@opengridcomputing.com> */
10 /* Copyright (c) 2008-2019, IBM Corporation */
11 /* Copyright (c) 2017, Open Grid Computing, Inc. */
12
13 #include <linux/workqueue.h>
14 #include <trace/events/sock.h>
15
16 #include "erdma.h"
17 #include "erdma_cm.h"
18 #include "erdma_verbs.h"
19
20 static struct workqueue_struct *erdma_cm_wq;
21
22 static void erdma_cm_llp_state_change(struct sock *sk);
23 static void erdma_cm_llp_data_ready(struct sock *sk);
24 static void erdma_cm_llp_error_report(struct sock *sk);
25
erdma_sk_assign_cm_upcalls(struct sock * sk)26 static void erdma_sk_assign_cm_upcalls(struct sock *sk)
27 {
28 write_lock_bh(&sk->sk_callback_lock);
29 sk->sk_state_change = erdma_cm_llp_state_change;
30 sk->sk_data_ready = erdma_cm_llp_data_ready;
31 sk->sk_error_report = erdma_cm_llp_error_report;
32 write_unlock_bh(&sk->sk_callback_lock);
33 }
34
erdma_sk_save_upcalls(struct sock * sk)35 static void erdma_sk_save_upcalls(struct sock *sk)
36 {
37 struct erdma_cep *cep = sk_to_cep(sk);
38
39 write_lock_bh(&sk->sk_callback_lock);
40 cep->sk_state_change = sk->sk_state_change;
41 cep->sk_data_ready = sk->sk_data_ready;
42 cep->sk_error_report = sk->sk_error_report;
43 write_unlock_bh(&sk->sk_callback_lock);
44 }
45
erdma_sk_restore_upcalls(struct sock * sk,struct erdma_cep * cep)46 static void erdma_sk_restore_upcalls(struct sock *sk, struct erdma_cep *cep)
47 {
48 sk->sk_state_change = cep->sk_state_change;
49 sk->sk_data_ready = cep->sk_data_ready;
50 sk->sk_error_report = cep->sk_error_report;
51 sk->sk_user_data = NULL;
52 }
53
erdma_socket_disassoc(struct socket * s)54 static void erdma_socket_disassoc(struct socket *s)
55 {
56 struct sock *sk = s->sk;
57 struct erdma_cep *cep;
58
59 if (sk) {
60 write_lock_bh(&sk->sk_callback_lock);
61 cep = sk_to_cep(sk);
62 if (cep) {
63 erdma_sk_restore_upcalls(sk, cep);
64 erdma_cep_put(cep);
65 } else {
66 WARN_ON_ONCE(1);
67 }
68 write_unlock_bh(&sk->sk_callback_lock);
69 } else {
70 WARN_ON_ONCE(1);
71 }
72 }
73
erdma_cep_socket_assoc(struct erdma_cep * cep,struct socket * s)74 static void erdma_cep_socket_assoc(struct erdma_cep *cep, struct socket *s)
75 {
76 cep->sock = s;
77 erdma_cep_get(cep);
78 s->sk->sk_user_data = cep;
79
80 erdma_sk_save_upcalls(s->sk);
81 erdma_sk_assign_cm_upcalls(s->sk);
82 }
83
erdma_disassoc_listen_cep(struct erdma_cep * cep)84 static void erdma_disassoc_listen_cep(struct erdma_cep *cep)
85 {
86 if (cep->listen_cep) {
87 erdma_cep_put(cep->listen_cep);
88 cep->listen_cep = NULL;
89 }
90 }
91
erdma_cep_alloc(struct erdma_dev * dev)92 static struct erdma_cep *erdma_cep_alloc(struct erdma_dev *dev)
93 {
94 struct erdma_cep *cep = kzalloc(sizeof(*cep), GFP_KERNEL);
95 unsigned long flags;
96
97 if (!cep)
98 return NULL;
99
100 INIT_LIST_HEAD(&cep->listenq);
101 INIT_LIST_HEAD(&cep->devq);
102 INIT_LIST_HEAD(&cep->work_freelist);
103
104 kref_init(&cep->ref);
105 cep->state = ERDMA_EPSTATE_IDLE;
106 init_waitqueue_head(&cep->waitq);
107 spin_lock_init(&cep->lock);
108 cep->dev = dev;
109
110 spin_lock_irqsave(&dev->lock, flags);
111 list_add_tail(&cep->devq, &dev->cep_list);
112 spin_unlock_irqrestore(&dev->lock, flags);
113
114 return cep;
115 }
116
erdma_cm_free_work(struct erdma_cep * cep)117 static void erdma_cm_free_work(struct erdma_cep *cep)
118 {
119 struct list_head *w, *tmp;
120 struct erdma_cm_work *work;
121
122 list_for_each_safe(w, tmp, &cep->work_freelist) {
123 work = list_entry(w, struct erdma_cm_work, list);
124 list_del(&work->list);
125 kfree(work);
126 }
127 }
128
erdma_cancel_mpatimer(struct erdma_cep * cep)129 static void erdma_cancel_mpatimer(struct erdma_cep *cep)
130 {
131 spin_lock_bh(&cep->lock);
132 if (cep->mpa_timer) {
133 if (cancel_delayed_work(&cep->mpa_timer->work)) {
134 erdma_cep_put(cep);
135 kfree(cep->mpa_timer);
136 }
137 cep->mpa_timer = NULL;
138 }
139 spin_unlock_bh(&cep->lock);
140 }
141
erdma_put_work(struct erdma_cm_work * work)142 static void erdma_put_work(struct erdma_cm_work *work)
143 {
144 INIT_LIST_HEAD(&work->list);
145 spin_lock_bh(&work->cep->lock);
146 list_add(&work->list, &work->cep->work_freelist);
147 spin_unlock_bh(&work->cep->lock);
148 }
149
erdma_cep_set_inuse(struct erdma_cep * cep)150 static void erdma_cep_set_inuse(struct erdma_cep *cep)
151 {
152 unsigned long flags;
153
154 spin_lock_irqsave(&cep->lock, flags);
155 while (cep->in_use) {
156 spin_unlock_irqrestore(&cep->lock, flags);
157 wait_event_interruptible(cep->waitq, !cep->in_use);
158 if (signal_pending(current))
159 flush_signals(current);
160
161 spin_lock_irqsave(&cep->lock, flags);
162 }
163
164 cep->in_use = 1;
165 spin_unlock_irqrestore(&cep->lock, flags);
166 }
167
erdma_cep_set_free(struct erdma_cep * cep)168 static void erdma_cep_set_free(struct erdma_cep *cep)
169 {
170 unsigned long flags;
171
172 spin_lock_irqsave(&cep->lock, flags);
173 cep->in_use = 0;
174 spin_unlock_irqrestore(&cep->lock, flags);
175
176 wake_up(&cep->waitq);
177 }
178
__erdma_cep_dealloc(struct kref * ref)179 static void __erdma_cep_dealloc(struct kref *ref)
180 {
181 struct erdma_cep *cep = container_of(ref, struct erdma_cep, ref);
182 struct erdma_dev *dev = cep->dev;
183 unsigned long flags;
184
185 WARN_ON(cep->listen_cep);
186
187 kfree(cep->private_data);
188 kfree(cep->mpa.pdata);
189 spin_lock_bh(&cep->lock);
190 if (!list_empty(&cep->work_freelist))
191 erdma_cm_free_work(cep);
192 spin_unlock_bh(&cep->lock);
193
194 spin_lock_irqsave(&dev->lock, flags);
195 list_del(&cep->devq);
196 spin_unlock_irqrestore(&dev->lock, flags);
197 kfree(cep);
198 }
199
erdma_get_work(struct erdma_cep * cep)200 static struct erdma_cm_work *erdma_get_work(struct erdma_cep *cep)
201 {
202 struct erdma_cm_work *work = NULL;
203
204 spin_lock_bh(&cep->lock);
205 if (!list_empty(&cep->work_freelist)) {
206 work = list_entry(cep->work_freelist.next, struct erdma_cm_work,
207 list);
208 list_del_init(&work->list);
209 }
210
211 spin_unlock_bh(&cep->lock);
212 return work;
213 }
214
erdma_cm_alloc_work(struct erdma_cep * cep,int num)215 static int erdma_cm_alloc_work(struct erdma_cep *cep, int num)
216 {
217 struct erdma_cm_work *work;
218
219 while (num--) {
220 work = kmalloc(sizeof(*work), GFP_KERNEL);
221 if (!work) {
222 if (!(list_empty(&cep->work_freelist)))
223 erdma_cm_free_work(cep);
224 return -ENOMEM;
225 }
226 work->cep = cep;
227 INIT_LIST_HEAD(&work->list);
228 list_add(&work->list, &cep->work_freelist);
229 }
230
231 return 0;
232 }
233
erdma_cm_upcall(struct erdma_cep * cep,enum iw_cm_event_type reason,int status)234 static int erdma_cm_upcall(struct erdma_cep *cep, enum iw_cm_event_type reason,
235 int status)
236 {
237 struct iw_cm_event event;
238 struct iw_cm_id *cm_id;
239
240 memset(&event, 0, sizeof(event));
241 event.status = status;
242 event.event = reason;
243
244 if (reason == IW_CM_EVENT_CONNECT_REQUEST) {
245 event.provider_data = cep;
246 cm_id = cep->listen_cep->cm_id;
247
248 event.ird = cep->dev->attrs.max_ird;
249 event.ord = cep->dev->attrs.max_ord;
250 } else {
251 cm_id = cep->cm_id;
252 }
253
254 if (reason == IW_CM_EVENT_CONNECT_REQUEST ||
255 reason == IW_CM_EVENT_CONNECT_REPLY) {
256 u16 pd_len = be16_to_cpu(cep->mpa.hdr.params.pd_len);
257
258 if (pd_len && cep->mpa.pdata) {
259 event.private_data_len = pd_len;
260 event.private_data = cep->mpa.pdata;
261 }
262
263 getname_local(cep->sock, &event.local_addr);
264 getname_peer(cep->sock, &event.remote_addr);
265 }
266
267 return cm_id->event_handler(cm_id, &event);
268 }
269
erdma_qp_cm_drop(struct erdma_qp * qp)270 void erdma_qp_cm_drop(struct erdma_qp *qp)
271 {
272 struct erdma_cep *cep = qp->cep;
273
274 if (!qp->cep)
275 return;
276
277 erdma_cep_set_inuse(cep);
278
279 /* already closed. */
280 if (cep->state == ERDMA_EPSTATE_CLOSED)
281 goto out;
282
283 if (cep->cm_id) {
284 switch (cep->state) {
285 case ERDMA_EPSTATE_AWAIT_MPAREP:
286 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY,
287 -EINVAL);
288 break;
289 case ERDMA_EPSTATE_RDMA_MODE:
290 erdma_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0);
291 break;
292 case ERDMA_EPSTATE_IDLE:
293 case ERDMA_EPSTATE_LISTENING:
294 case ERDMA_EPSTATE_CONNECTING:
295 case ERDMA_EPSTATE_AWAIT_MPAREQ:
296 case ERDMA_EPSTATE_RECVD_MPAREQ:
297 case ERDMA_EPSTATE_CLOSED:
298 default:
299 break;
300 }
301 cep->cm_id->rem_ref(cep->cm_id);
302 cep->cm_id = NULL;
303 erdma_cep_put(cep);
304 }
305 cep->state = ERDMA_EPSTATE_CLOSED;
306
307 if (cep->sock) {
308 erdma_socket_disassoc(cep->sock);
309 sock_release(cep->sock);
310 cep->sock = NULL;
311 }
312
313 if (cep->qp) {
314 cep->qp = NULL;
315 erdma_qp_put(qp);
316 }
317 out:
318 erdma_cep_set_free(cep);
319 }
320
erdma_cep_put(struct erdma_cep * cep)321 void erdma_cep_put(struct erdma_cep *cep)
322 {
323 WARN_ON(kref_read(&cep->ref) < 1);
324 kref_put(&cep->ref, __erdma_cep_dealloc);
325 }
326
erdma_cep_get(struct erdma_cep * cep)327 void erdma_cep_get(struct erdma_cep *cep)
328 {
329 kref_get(&cep->ref);
330 }
331
erdma_send_mpareqrep(struct erdma_cep * cep,const void * pdata,u8 pd_len)332 static int erdma_send_mpareqrep(struct erdma_cep *cep, const void *pdata,
333 u8 pd_len)
334 {
335 struct socket *s = cep->sock;
336 struct mpa_rr *rr = &cep->mpa.hdr;
337 struct kvec iov[3];
338 struct msghdr msg;
339 int iovec_num = 0;
340 int ret;
341 int mpa_len;
342
343 memset(&msg, 0, sizeof(msg));
344
345 rr->params.pd_len = cpu_to_be16(pd_len);
346
347 iov[iovec_num].iov_base = rr;
348 iov[iovec_num].iov_len = sizeof(*rr);
349 iovec_num++;
350 mpa_len = sizeof(*rr);
351
352 iov[iovec_num].iov_base = &cep->mpa.ext_data;
353 iov[iovec_num].iov_len = sizeof(cep->mpa.ext_data);
354 iovec_num++;
355 mpa_len += sizeof(cep->mpa.ext_data);
356
357 if (pd_len) {
358 iov[iovec_num].iov_base = (char *)pdata;
359 iov[iovec_num].iov_len = pd_len;
360 mpa_len += pd_len;
361 iovec_num++;
362 }
363
364 ret = kernel_sendmsg(s, &msg, iov, iovec_num, mpa_len);
365
366 return ret < 0 ? ret : 0;
367 }
368
ksock_recv(struct socket * sock,char * buf,size_t size,int flags)369 static inline int ksock_recv(struct socket *sock, char *buf, size_t size,
370 int flags)
371 {
372 struct kvec iov = { buf, size };
373 struct msghdr msg = { .msg_name = NULL, .msg_flags = flags };
374
375 return kernel_recvmsg(sock, &msg, &iov, 1, size, flags);
376 }
377
__recv_mpa_hdr(struct erdma_cep * cep,int hdr_rcvd,char * hdr,int hdr_size,int * rcvd_out)378 static int __recv_mpa_hdr(struct erdma_cep *cep, int hdr_rcvd, char *hdr,
379 int hdr_size, int *rcvd_out)
380 {
381 struct socket *s = cep->sock;
382 int rcvd;
383
384 *rcvd_out = 0;
385 if (hdr_rcvd < hdr_size) {
386 rcvd = ksock_recv(s, hdr + hdr_rcvd, hdr_size - hdr_rcvd,
387 MSG_DONTWAIT);
388 if (rcvd == -EAGAIN)
389 return -EAGAIN;
390
391 if (rcvd <= 0)
392 return -ECONNABORTED;
393
394 hdr_rcvd += rcvd;
395 *rcvd_out = rcvd;
396
397 if (hdr_rcvd < hdr_size)
398 return -EAGAIN;
399 }
400
401 return 0;
402 }
403
__mpa_rr_set_revision(__be16 * bits,u8 rev)404 static void __mpa_rr_set_revision(__be16 *bits, u8 rev)
405 {
406 *bits = (*bits & ~MPA_RR_MASK_REVISION) |
407 (cpu_to_be16(rev) & MPA_RR_MASK_REVISION);
408 }
409
__mpa_rr_revision(__be16 mpa_rr_bits)410 static u8 __mpa_rr_revision(__be16 mpa_rr_bits)
411 {
412 __be16 rev = mpa_rr_bits & MPA_RR_MASK_REVISION;
413
414 return (u8)be16_to_cpu(rev);
415 }
416
__mpa_ext_set_cc(__be32 * bits,u32 cc)417 static void __mpa_ext_set_cc(__be32 *bits, u32 cc)
418 {
419 *bits = (*bits & ~MPA_EXT_FLAG_CC) |
420 (cpu_to_be32(cc) & MPA_EXT_FLAG_CC);
421 }
422
__mpa_ext_cc(__be32 mpa_ext_bits)423 static u8 __mpa_ext_cc(__be32 mpa_ext_bits)
424 {
425 __be32 cc = mpa_ext_bits & MPA_EXT_FLAG_CC;
426
427 return (u8)be32_to_cpu(cc);
428 }
429
430 /*
431 * Receive MPA Request/Reply header.
432 *
433 * Returns 0 if complete MPA Request/Reply haeder including
434 * eventual private data was received. Returns -EAGAIN if
435 * header was partially received or negative error code otherwise.
436 *
437 * Context: May be called in process context only
438 */
erdma_recv_mpa_rr(struct erdma_cep * cep)439 static int erdma_recv_mpa_rr(struct erdma_cep *cep)
440 {
441 struct mpa_rr *hdr = &cep->mpa.hdr;
442 struct socket *s = cep->sock;
443 u16 pd_len;
444 int rcvd, to_rcv, ret, pd_rcvd;
445
446 if (cep->mpa.bytes_rcvd < sizeof(struct mpa_rr)) {
447 ret = __recv_mpa_hdr(cep, cep->mpa.bytes_rcvd,
448 (char *)&cep->mpa.hdr,
449 sizeof(struct mpa_rr), &rcvd);
450 cep->mpa.bytes_rcvd += rcvd;
451 if (ret)
452 return ret;
453 }
454
455 if (be16_to_cpu(hdr->params.pd_len) > MPA_MAX_PRIVDATA ||
456 __mpa_rr_revision(hdr->params.bits) != MPA_REVISION_EXT_1)
457 return -EPROTO;
458
459 if (cep->mpa.bytes_rcvd - sizeof(struct mpa_rr) <
460 sizeof(struct erdma_mpa_ext)) {
461 ret = __recv_mpa_hdr(
462 cep, cep->mpa.bytes_rcvd - sizeof(struct mpa_rr),
463 (char *)&cep->mpa.ext_data,
464 sizeof(struct erdma_mpa_ext), &rcvd);
465 cep->mpa.bytes_rcvd += rcvd;
466 if (ret)
467 return ret;
468 }
469
470 pd_len = be16_to_cpu(hdr->params.pd_len);
471 pd_rcvd = cep->mpa.bytes_rcvd - sizeof(struct mpa_rr) -
472 sizeof(struct erdma_mpa_ext);
473 to_rcv = pd_len - pd_rcvd;
474
475 if (!to_rcv) {
476 /*
477 * We have received the whole MPA Request/Reply message.
478 * Check against peer protocol violation.
479 */
480 u32 word;
481
482 ret = __recv_mpa_hdr(cep, 0, (char *)&word, sizeof(word),
483 &rcvd);
484 if (ret == -EAGAIN && rcvd == 0)
485 return 0;
486
487 if (ret)
488 return ret;
489
490 return -EPROTO;
491 }
492
493 /*
494 * At this point, MPA header has been fully received, and pd_len != 0.
495 * So, begin to receive private data.
496 */
497 if (!cep->mpa.pdata) {
498 cep->mpa.pdata = kmalloc(pd_len + 4, GFP_KERNEL);
499 if (!cep->mpa.pdata)
500 return -ENOMEM;
501 }
502
503 rcvd = ksock_recv(s, cep->mpa.pdata + pd_rcvd, to_rcv + 4,
504 MSG_DONTWAIT);
505 if (rcvd < 0)
506 return rcvd;
507
508 if (rcvd > to_rcv)
509 return -EPROTO;
510
511 cep->mpa.bytes_rcvd += rcvd;
512
513 if (to_rcv == rcvd)
514 return 0;
515
516 return -EAGAIN;
517 }
518
519 /*
520 * erdma_proc_mpareq()
521 *
522 * Read MPA Request from socket and signal new connection to IWCM
523 * if success. Caller must hold lock on corresponding listening CEP.
524 */
erdma_proc_mpareq(struct erdma_cep * cep)525 static int erdma_proc_mpareq(struct erdma_cep *cep)
526 {
527 struct mpa_rr *req;
528 int ret;
529
530 ret = erdma_recv_mpa_rr(cep);
531 if (ret)
532 return ret;
533
534 req = &cep->mpa.hdr;
535
536 if (memcmp(req->key, MPA_KEY_REQ, MPA_KEY_SIZE))
537 return -EPROTO;
538
539 memcpy(req->key, MPA_KEY_REP, MPA_KEY_SIZE);
540
541 /* Currently does not support marker and crc. */
542 if (req->params.bits & MPA_RR_FLAG_MARKERS ||
543 req->params.bits & MPA_RR_FLAG_CRC)
544 goto reject_conn;
545
546 cep->state = ERDMA_EPSTATE_RECVD_MPAREQ;
547
548 /* Keep reference until IWCM accepts/rejects */
549 erdma_cep_get(cep);
550 ret = erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REQUEST, 0);
551 if (ret)
552 erdma_cep_put(cep);
553
554 return ret;
555
556 reject_conn:
557 req->params.bits &= ~MPA_RR_FLAG_MARKERS;
558 req->params.bits |= MPA_RR_FLAG_REJECT;
559 req->params.bits &= ~MPA_RR_FLAG_CRC;
560
561 kfree(cep->mpa.pdata);
562 cep->mpa.pdata = NULL;
563 erdma_send_mpareqrep(cep, NULL, 0);
564
565 return -EOPNOTSUPP;
566 }
567
erdma_proc_mpareply(struct erdma_cep * cep)568 static int erdma_proc_mpareply(struct erdma_cep *cep)
569 {
570 struct erdma_qp_attrs qp_attrs;
571 struct erdma_qp *qp = cep->qp;
572 struct mpa_rr *rep;
573 int ret;
574
575 ret = erdma_recv_mpa_rr(cep);
576 if (ret)
577 goto out_err;
578
579 erdma_cancel_mpatimer(cep);
580
581 rep = &cep->mpa.hdr;
582
583 if (memcmp(rep->key, MPA_KEY_REP, MPA_KEY_SIZE)) {
584 ret = -EPROTO;
585 goto out_err;
586 }
587
588 if (rep->params.bits & MPA_RR_FLAG_REJECT) {
589 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -ECONNRESET);
590 return -ECONNRESET;
591 }
592
593 /* Currently does not support marker and crc. */
594 if ((rep->params.bits & MPA_RR_FLAG_MARKERS) ||
595 (rep->params.bits & MPA_RR_FLAG_CRC)) {
596 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -ECONNREFUSED);
597 return -EINVAL;
598 }
599
600 memset(&qp_attrs, 0, sizeof(qp_attrs));
601 qp_attrs.irq_size = cep->ird;
602 qp_attrs.orq_size = cep->ord;
603 qp_attrs.state = ERDMA_QP_STATE_RTS;
604
605 down_write(&qp->state_lock);
606 if (qp->attrs.state > ERDMA_QP_STATE_RTR) {
607 ret = -EINVAL;
608 up_write(&qp->state_lock);
609 goto out_err;
610 }
611
612 qp->attrs.qp_type = ERDMA_QP_ACTIVE;
613 if (__mpa_ext_cc(cep->mpa.ext_data.bits) != qp->attrs.cc)
614 qp->attrs.cc = COMPROMISE_CC;
615
616 ret = erdma_modify_qp_internal(qp, &qp_attrs,
617 ERDMA_QP_ATTR_STATE |
618 ERDMA_QP_ATTR_LLP_HANDLE |
619 ERDMA_QP_ATTR_MPA);
620
621 up_write(&qp->state_lock);
622
623 if (!ret) {
624 ret = erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, 0);
625 if (!ret)
626 cep->state = ERDMA_EPSTATE_RDMA_MODE;
627
628 return 0;
629 }
630
631 out_err:
632 if (ret != -EAGAIN)
633 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -EINVAL);
634
635 return ret;
636 }
637
erdma_accept_newconn(struct erdma_cep * cep)638 static void erdma_accept_newconn(struct erdma_cep *cep)
639 {
640 struct socket *s = cep->sock;
641 struct socket *new_s = NULL;
642 struct erdma_cep *new_cep = NULL;
643 int ret = 0;
644
645 if (cep->state != ERDMA_EPSTATE_LISTENING)
646 goto error;
647
648 new_cep = erdma_cep_alloc(cep->dev);
649 if (!new_cep)
650 goto error;
651
652 /*
653 * 4: Allocate a sufficient number of work elements
654 * to allow concurrent handling of local + peer close
655 * events, MPA header processing + MPA timeout.
656 */
657 if (erdma_cm_alloc_work(new_cep, 4) != 0)
658 goto error;
659
660 /*
661 * Copy saved socket callbacks from listening CEP
662 * and assign new socket with new CEP
663 */
664 new_cep->sk_state_change = cep->sk_state_change;
665 new_cep->sk_data_ready = cep->sk_data_ready;
666 new_cep->sk_error_report = cep->sk_error_report;
667
668 ret = kernel_accept(s, &new_s, O_NONBLOCK);
669 if (ret != 0)
670 goto error;
671
672 new_cep->sock = new_s;
673 erdma_cep_get(new_cep);
674 new_s->sk->sk_user_data = new_cep;
675
676 tcp_sock_set_nodelay(new_s->sk);
677 new_cep->state = ERDMA_EPSTATE_AWAIT_MPAREQ;
678
679 ret = erdma_cm_queue_work(new_cep, ERDMA_CM_WORK_MPATIMEOUT);
680 if (ret)
681 goto error;
682
683 new_cep->listen_cep = cep;
684 erdma_cep_get(cep);
685
686 if (atomic_read(&new_s->sk->sk_rmem_alloc)) {
687 /* MPA REQ already queued */
688 erdma_cep_set_inuse(new_cep);
689 ret = erdma_proc_mpareq(new_cep);
690 if (ret != -EAGAIN) {
691 erdma_cep_put(cep);
692 new_cep->listen_cep = NULL;
693 if (ret) {
694 erdma_cep_set_free(new_cep);
695 goto error;
696 }
697 }
698 erdma_cep_set_free(new_cep);
699 }
700 return;
701
702 error:
703 if (new_cep) {
704 new_cep->state = ERDMA_EPSTATE_CLOSED;
705 erdma_cancel_mpatimer(new_cep);
706
707 erdma_cep_put(new_cep);
708 }
709
710 if (new_s) {
711 erdma_socket_disassoc(new_s);
712 sock_release(new_s);
713 }
714 }
715
erdma_newconn_connected(struct erdma_cep * cep)716 static int erdma_newconn_connected(struct erdma_cep *cep)
717 {
718 int ret = 0;
719
720 cep->mpa.hdr.params.bits = 0;
721 __mpa_rr_set_revision(&cep->mpa.hdr.params.bits, MPA_REVISION_EXT_1);
722
723 memcpy(cep->mpa.hdr.key, MPA_KEY_REQ, MPA_KEY_SIZE);
724 cep->mpa.ext_data.cookie = cpu_to_be32(cep->qp->attrs.cookie);
725 __mpa_ext_set_cc(&cep->mpa.ext_data.bits, cep->qp->attrs.cc);
726
727 ret = erdma_send_mpareqrep(cep, cep->private_data, cep->pd_len);
728 cep->state = ERDMA_EPSTATE_AWAIT_MPAREP;
729 cep->mpa.hdr.params.pd_len = 0;
730
731 if (ret >= 0)
732 ret = erdma_cm_queue_work(cep, ERDMA_CM_WORK_MPATIMEOUT);
733
734 return ret;
735 }
736
erdma_cm_work_handler(struct work_struct * w)737 static void erdma_cm_work_handler(struct work_struct *w)
738 {
739 struct erdma_cm_work *work;
740 struct erdma_cep *cep;
741 int release_cep = 0, ret = 0;
742
743 work = container_of(w, struct erdma_cm_work, work.work);
744 cep = work->cep;
745
746 erdma_cep_set_inuse(cep);
747
748 switch (work->type) {
749 case ERDMA_CM_WORK_CONNECTED:
750 erdma_cancel_mpatimer(cep);
751 if (cep->state == ERDMA_EPSTATE_CONNECTING) {
752 ret = erdma_newconn_connected(cep);
753 if (ret) {
754 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY,
755 -EIO);
756 release_cep = 1;
757 }
758 }
759 break;
760 case ERDMA_CM_WORK_CONNECTTIMEOUT:
761 if (cep->state == ERDMA_EPSTATE_CONNECTING) {
762 cep->mpa_timer = NULL;
763 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY,
764 -ETIMEDOUT);
765 release_cep = 1;
766 }
767 break;
768 case ERDMA_CM_WORK_ACCEPT:
769 erdma_accept_newconn(cep);
770 break;
771 case ERDMA_CM_WORK_READ_MPAHDR:
772 if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREQ) {
773 if (cep->listen_cep) {
774 erdma_cep_set_inuse(cep->listen_cep);
775
776 if (cep->listen_cep->state ==
777 ERDMA_EPSTATE_LISTENING)
778 ret = erdma_proc_mpareq(cep);
779 else
780 ret = -EFAULT;
781
782 erdma_cep_set_free(cep->listen_cep);
783
784 if (ret != -EAGAIN) {
785 erdma_cep_put(cep->listen_cep);
786 cep->listen_cep = NULL;
787 if (ret)
788 erdma_cep_put(cep);
789 }
790 }
791 } else if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREP) {
792 ret = erdma_proc_mpareply(cep);
793 }
794
795 if (ret && ret != -EAGAIN)
796 release_cep = 1;
797 break;
798 case ERDMA_CM_WORK_CLOSE_LLP:
799 if (cep->cm_id)
800 erdma_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0);
801 release_cep = 1;
802 break;
803 case ERDMA_CM_WORK_PEER_CLOSE:
804 if (cep->cm_id) {
805 if (cep->state == ERDMA_EPSTATE_CONNECTING ||
806 cep->state == ERDMA_EPSTATE_AWAIT_MPAREP) {
807 /*
808 * MPA reply not received, but connection drop
809 */
810 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY,
811 -ECONNRESET);
812 } else if (cep->state == ERDMA_EPSTATE_RDMA_MODE) {
813 /*
814 * NOTE: IW_CM_EVENT_DISCONNECT is given just
815 * to transition IWCM into CLOSING.
816 */
817 erdma_cm_upcall(cep, IW_CM_EVENT_DISCONNECT, 0);
818 erdma_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0);
819 }
820 } else if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREQ) {
821 /* Socket close before MPA request received. */
822 erdma_disassoc_listen_cep(cep);
823 erdma_cep_put(cep);
824 }
825 release_cep = 1;
826 break;
827 case ERDMA_CM_WORK_MPATIMEOUT:
828 cep->mpa_timer = NULL;
829 if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREP) {
830 /*
831 * MPA request timed out:
832 * Hide any partially received private data and signal
833 * timeout
834 */
835 cep->mpa.hdr.params.pd_len = 0;
836
837 if (cep->cm_id)
838 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY,
839 -ETIMEDOUT);
840 release_cep = 1;
841 } else if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREQ) {
842 /* No MPA req received after peer TCP stream setup. */
843 erdma_disassoc_listen_cep(cep);
844
845 erdma_cep_put(cep);
846 release_cep = 1;
847 }
848 break;
849 default:
850 WARN(1, "Undefined CM work type: %d\n", work->type);
851 }
852
853 if (release_cep) {
854 erdma_cancel_mpatimer(cep);
855 cep->state = ERDMA_EPSTATE_CLOSED;
856 if (cep->qp) {
857 struct erdma_qp *qp = cep->qp;
858 /*
859 * Serialize a potential race with application
860 * closing the QP and calling erdma_qp_cm_drop()
861 */
862 erdma_qp_get(qp);
863 erdma_cep_set_free(cep);
864
865 erdma_qp_llp_close(qp);
866 erdma_qp_put(qp);
867
868 erdma_cep_set_inuse(cep);
869 cep->qp = NULL;
870 erdma_qp_put(qp);
871 }
872
873 if (cep->sock) {
874 erdma_socket_disassoc(cep->sock);
875 sock_release(cep->sock);
876 cep->sock = NULL;
877 }
878
879 if (cep->cm_id) {
880 cep->cm_id->rem_ref(cep->cm_id);
881 cep->cm_id = NULL;
882 if (cep->state != ERDMA_EPSTATE_LISTENING)
883 erdma_cep_put(cep);
884 }
885 }
886 erdma_cep_set_free(cep);
887 erdma_put_work(work);
888 erdma_cep_put(cep);
889 }
890
erdma_cm_queue_work(struct erdma_cep * cep,enum erdma_work_type type)891 int erdma_cm_queue_work(struct erdma_cep *cep, enum erdma_work_type type)
892 {
893 struct erdma_cm_work *work = erdma_get_work(cep);
894 unsigned long delay = 0;
895
896 if (!work)
897 return -ENOMEM;
898
899 work->type = type;
900 work->cep = cep;
901
902 erdma_cep_get(cep);
903
904 INIT_DELAYED_WORK(&work->work, erdma_cm_work_handler);
905
906 if (type == ERDMA_CM_WORK_MPATIMEOUT) {
907 cep->mpa_timer = work;
908
909 if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREP)
910 delay = MPAREP_TIMEOUT;
911 else
912 delay = MPAREQ_TIMEOUT;
913 } else if (type == ERDMA_CM_WORK_CONNECTTIMEOUT) {
914 cep->mpa_timer = work;
915
916 delay = CONNECT_TIMEOUT;
917 }
918
919 queue_delayed_work(erdma_cm_wq, &work->work, delay);
920
921 return 0;
922 }
923
erdma_cm_llp_data_ready(struct sock * sk)924 static void erdma_cm_llp_data_ready(struct sock *sk)
925 {
926 struct erdma_cep *cep;
927
928 trace_sk_data_ready(sk);
929
930 read_lock(&sk->sk_callback_lock);
931
932 cep = sk_to_cep(sk);
933 if (!cep)
934 goto out;
935
936 if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREQ ||
937 cep->state == ERDMA_EPSTATE_AWAIT_MPAREP)
938 erdma_cm_queue_work(cep, ERDMA_CM_WORK_READ_MPAHDR);
939
940 out:
941 read_unlock(&sk->sk_callback_lock);
942 }
943
erdma_cm_llp_error_report(struct sock * sk)944 static void erdma_cm_llp_error_report(struct sock *sk)
945 {
946 struct erdma_cep *cep = sk_to_cep(sk);
947
948 if (cep)
949 cep->sk_error_report(sk);
950 }
951
erdma_cm_llp_state_change(struct sock * sk)952 static void erdma_cm_llp_state_change(struct sock *sk)
953 {
954 struct erdma_cep *cep;
955 void (*orig_state_change)(struct sock *sk);
956
957 read_lock(&sk->sk_callback_lock);
958
959 cep = sk_to_cep(sk);
960 if (!cep) {
961 read_unlock(&sk->sk_callback_lock);
962 return;
963 }
964 orig_state_change = cep->sk_state_change;
965
966 switch (sk->sk_state) {
967 case TCP_ESTABLISHED:
968 if (cep->state == ERDMA_EPSTATE_CONNECTING)
969 erdma_cm_queue_work(cep, ERDMA_CM_WORK_CONNECTED);
970 else
971 erdma_cm_queue_work(cep, ERDMA_CM_WORK_ACCEPT);
972 break;
973 case TCP_CLOSE:
974 case TCP_CLOSE_WAIT:
975 if (cep->state != ERDMA_EPSTATE_LISTENING)
976 erdma_cm_queue_work(cep, ERDMA_CM_WORK_PEER_CLOSE);
977 break;
978 default:
979 break;
980 }
981 read_unlock(&sk->sk_callback_lock);
982 orig_state_change(sk);
983 }
984
kernel_bindconnect(struct socket * s,struct sockaddr * laddr,int laddrlen,struct sockaddr * raddr,int raddrlen,int flags)985 static int kernel_bindconnect(struct socket *s, struct sockaddr *laddr,
986 int laddrlen, struct sockaddr *raddr,
987 int raddrlen, int flags)
988 {
989 int ret;
990
991 sock_set_reuseaddr(s->sk);
992 ret = s->ops->bind(s, laddr, laddrlen);
993 if (ret)
994 return ret;
995 ret = s->ops->connect(s, raddr, raddrlen, flags);
996 return ret < 0 ? ret : 0;
997 }
998
erdma_connect(struct iw_cm_id * id,struct iw_cm_conn_param * params)999 int erdma_connect(struct iw_cm_id *id, struct iw_cm_conn_param *params)
1000 {
1001 struct erdma_dev *dev = to_edev(id->device);
1002 struct erdma_qp *qp;
1003 struct erdma_cep *cep = NULL;
1004 struct socket *s = NULL;
1005 struct sockaddr *laddr = (struct sockaddr *)&id->m_local_addr;
1006 struct sockaddr *raddr = (struct sockaddr *)&id->m_remote_addr;
1007 u16 pd_len = params->private_data_len;
1008 int ret;
1009
1010 if (pd_len > MPA_MAX_PRIVDATA)
1011 return -EINVAL;
1012
1013 if (params->ird > dev->attrs.max_ird ||
1014 params->ord > dev->attrs.max_ord)
1015 return -EINVAL;
1016
1017 if (laddr->sa_family != AF_INET || raddr->sa_family != AF_INET)
1018 return -EAFNOSUPPORT;
1019
1020 qp = find_qp_by_qpn(dev, params->qpn);
1021 if (!qp)
1022 return -ENOENT;
1023 erdma_qp_get(qp);
1024
1025 ret = sock_create(AF_INET, SOCK_STREAM, IPPROTO_TCP, &s);
1026 if (ret < 0)
1027 goto error_put_qp;
1028
1029 cep = erdma_cep_alloc(dev);
1030 if (!cep) {
1031 ret = -ENOMEM;
1032 goto error_release_sock;
1033 }
1034
1035 erdma_cep_set_inuse(cep);
1036
1037 /* Associate QP with CEP */
1038 erdma_cep_get(cep);
1039 qp->cep = cep;
1040 cep->qp = qp;
1041
1042 /* Associate cm_id with CEP */
1043 id->add_ref(id);
1044 cep->cm_id = id;
1045
1046 /*
1047 * 6: Allocate a sufficient number of work elements
1048 * to allow concurrent handling of local + peer close
1049 * events, MPA header processing + MPA timeout, connected event
1050 * and connect timeout.
1051 */
1052 ret = erdma_cm_alloc_work(cep, 6);
1053 if (ret != 0) {
1054 ret = -ENOMEM;
1055 goto error_release_cep;
1056 }
1057
1058 cep->ird = params->ird;
1059 cep->ord = params->ord;
1060 cep->state = ERDMA_EPSTATE_CONNECTING;
1061
1062 erdma_cep_socket_assoc(cep, s);
1063
1064 if (pd_len) {
1065 cep->pd_len = pd_len;
1066 cep->private_data = kmalloc(pd_len, GFP_KERNEL);
1067 if (!cep->private_data) {
1068 ret = -ENOMEM;
1069 goto error_disassoc;
1070 }
1071
1072 memcpy(cep->private_data, params->private_data,
1073 params->private_data_len);
1074 }
1075
1076 ret = kernel_bindconnect(s, laddr, sizeof(*laddr), raddr,
1077 sizeof(*raddr), O_NONBLOCK);
1078 if (ret != -EINPROGRESS && ret != 0) {
1079 goto error_disassoc;
1080 } else if (ret == 0) {
1081 ret = erdma_cm_queue_work(cep, ERDMA_CM_WORK_CONNECTED);
1082 if (ret)
1083 goto error_disassoc;
1084 } else {
1085 ret = erdma_cm_queue_work(cep, ERDMA_CM_WORK_CONNECTTIMEOUT);
1086 if (ret)
1087 goto error_disassoc;
1088 }
1089
1090 erdma_cep_set_free(cep);
1091 return 0;
1092
1093 error_disassoc:
1094 kfree(cep->private_data);
1095 cep->private_data = NULL;
1096 cep->pd_len = 0;
1097
1098 erdma_socket_disassoc(s);
1099
1100 error_release_cep:
1101 /* disassoc with cm_id */
1102 cep->cm_id = NULL;
1103 id->rem_ref(id);
1104
1105 /* disassoc with qp */
1106 qp->cep = NULL;
1107 erdma_cep_put(cep);
1108 cep->qp = NULL;
1109
1110 cep->state = ERDMA_EPSTATE_CLOSED;
1111
1112 erdma_cep_set_free(cep);
1113
1114 /* release the cep. */
1115 erdma_cep_put(cep);
1116
1117 error_release_sock:
1118 if (s)
1119 sock_release(s);
1120 error_put_qp:
1121 erdma_qp_put(qp);
1122
1123 return ret;
1124 }
1125
erdma_accept(struct iw_cm_id * id,struct iw_cm_conn_param * params)1126 int erdma_accept(struct iw_cm_id *id, struct iw_cm_conn_param *params)
1127 {
1128 struct erdma_dev *dev = to_edev(id->device);
1129 struct erdma_cep *cep = (struct erdma_cep *)id->provider_data;
1130 struct erdma_qp *qp;
1131 struct erdma_qp_attrs qp_attrs;
1132 int ret;
1133
1134 erdma_cep_set_inuse(cep);
1135 erdma_cep_put(cep);
1136
1137 /* Free lingering inbound private data */
1138 if (cep->mpa.hdr.params.pd_len) {
1139 cep->mpa.hdr.params.pd_len = 0;
1140 kfree(cep->mpa.pdata);
1141 cep->mpa.pdata = NULL;
1142 }
1143 erdma_cancel_mpatimer(cep);
1144
1145 if (cep->state != ERDMA_EPSTATE_RECVD_MPAREQ) {
1146 erdma_cep_set_free(cep);
1147 erdma_cep_put(cep);
1148
1149 return -ECONNRESET;
1150 }
1151
1152 qp = find_qp_by_qpn(dev, params->qpn);
1153 if (!qp)
1154 return -ENOENT;
1155 erdma_qp_get(qp);
1156
1157 down_write(&qp->state_lock);
1158 if (qp->attrs.state > ERDMA_QP_STATE_RTR) {
1159 ret = -EINVAL;
1160 up_write(&qp->state_lock);
1161 goto error;
1162 }
1163
1164 if (params->ord > dev->attrs.max_ord ||
1165 params->ird > dev->attrs.max_ord) {
1166 ret = -EINVAL;
1167 up_write(&qp->state_lock);
1168 goto error;
1169 }
1170
1171 if (params->private_data_len > MPA_MAX_PRIVDATA) {
1172 ret = -EINVAL;
1173 up_write(&qp->state_lock);
1174 goto error;
1175 }
1176
1177 cep->ird = params->ird;
1178 cep->ord = params->ord;
1179
1180 cep->cm_id = id;
1181 id->add_ref(id);
1182
1183 memset(&qp_attrs, 0, sizeof(qp_attrs));
1184 qp_attrs.orq_size = params->ord;
1185 qp_attrs.irq_size = params->ird;
1186
1187 qp_attrs.state = ERDMA_QP_STATE_RTS;
1188
1189 /* Associate QP with CEP */
1190 erdma_cep_get(cep);
1191 qp->cep = cep;
1192 cep->qp = qp;
1193
1194 cep->state = ERDMA_EPSTATE_RDMA_MODE;
1195
1196 qp->attrs.qp_type = ERDMA_QP_PASSIVE;
1197 qp->attrs.pd_len = params->private_data_len;
1198
1199 if (qp->attrs.cc != __mpa_ext_cc(cep->mpa.ext_data.bits))
1200 qp->attrs.cc = COMPROMISE_CC;
1201
1202 /* move to rts */
1203 ret = erdma_modify_qp_internal(qp, &qp_attrs,
1204 ERDMA_QP_ATTR_STATE |
1205 ERDMA_QP_ATTR_ORD |
1206 ERDMA_QP_ATTR_LLP_HANDLE |
1207 ERDMA_QP_ATTR_IRD |
1208 ERDMA_QP_ATTR_MPA);
1209 up_write(&qp->state_lock);
1210
1211 if (ret)
1212 goto error;
1213
1214 cep->mpa.ext_data.bits = 0;
1215 __mpa_ext_set_cc(&cep->mpa.ext_data.bits, qp->attrs.cc);
1216 cep->mpa.ext_data.cookie = cpu_to_be32(cep->qp->attrs.cookie);
1217
1218 ret = erdma_send_mpareqrep(cep, params->private_data,
1219 params->private_data_len);
1220 if (!ret) {
1221 ret = erdma_cm_upcall(cep, IW_CM_EVENT_ESTABLISHED, 0);
1222 if (ret)
1223 goto error;
1224
1225 erdma_cep_set_free(cep);
1226
1227 return 0;
1228 }
1229
1230 error:
1231 erdma_socket_disassoc(cep->sock);
1232 sock_release(cep->sock);
1233 cep->sock = NULL;
1234
1235 cep->state = ERDMA_EPSTATE_CLOSED;
1236
1237 if (cep->cm_id) {
1238 cep->cm_id->rem_ref(id);
1239 cep->cm_id = NULL;
1240 }
1241
1242 if (qp->cep) {
1243 erdma_cep_put(cep);
1244 qp->cep = NULL;
1245 }
1246
1247 cep->qp = NULL;
1248 erdma_qp_put(qp);
1249
1250 erdma_cep_set_free(cep);
1251 erdma_cep_put(cep);
1252
1253 return ret;
1254 }
1255
erdma_reject(struct iw_cm_id * id,const void * pdata,u8 plen)1256 int erdma_reject(struct iw_cm_id *id, const void *pdata, u8 plen)
1257 {
1258 struct erdma_cep *cep = (struct erdma_cep *)id->provider_data;
1259
1260 erdma_cep_set_inuse(cep);
1261 erdma_cep_put(cep);
1262
1263 erdma_cancel_mpatimer(cep);
1264
1265 if (cep->state != ERDMA_EPSTATE_RECVD_MPAREQ) {
1266 erdma_cep_set_free(cep);
1267 erdma_cep_put(cep);
1268
1269 return -ECONNRESET;
1270 }
1271
1272 if (__mpa_rr_revision(cep->mpa.hdr.params.bits) == MPA_REVISION_EXT_1) {
1273 cep->mpa.hdr.params.bits |= MPA_RR_FLAG_REJECT; /* reject */
1274 erdma_send_mpareqrep(cep, pdata, plen);
1275 }
1276
1277 erdma_socket_disassoc(cep->sock);
1278 sock_release(cep->sock);
1279 cep->sock = NULL;
1280
1281 cep->state = ERDMA_EPSTATE_CLOSED;
1282
1283 erdma_cep_set_free(cep);
1284 erdma_cep_put(cep);
1285
1286 return 0;
1287 }
1288
erdma_create_listen(struct iw_cm_id * id,int backlog)1289 int erdma_create_listen(struct iw_cm_id *id, int backlog)
1290 {
1291 struct socket *s;
1292 struct erdma_cep *cep = NULL;
1293 int ret = 0;
1294 struct erdma_dev *dev = to_edev(id->device);
1295 int addr_family = id->local_addr.ss_family;
1296 struct sockaddr_in *laddr = &to_sockaddr_in(id->local_addr);
1297
1298 if (addr_family != AF_INET)
1299 return -EAFNOSUPPORT;
1300
1301 ret = sock_create(addr_family, SOCK_STREAM, IPPROTO_TCP, &s);
1302 if (ret < 0)
1303 return ret;
1304
1305 sock_set_reuseaddr(s->sk);
1306
1307 /* For wildcard addr, limit binding to current device only */
1308 if (ipv4_is_zeronet(laddr->sin_addr.s_addr))
1309 s->sk->sk_bound_dev_if = dev->netdev->ifindex;
1310
1311 ret = s->ops->bind(s, (struct sockaddr *)laddr,
1312 sizeof(struct sockaddr_in));
1313 if (ret)
1314 goto error;
1315
1316 cep = erdma_cep_alloc(dev);
1317 if (!cep) {
1318 ret = -ENOMEM;
1319 goto error;
1320 }
1321 erdma_cep_socket_assoc(cep, s);
1322
1323 ret = erdma_cm_alloc_work(cep, backlog);
1324 if (ret)
1325 goto error;
1326
1327 ret = s->ops->listen(s, backlog);
1328 if (ret)
1329 goto error;
1330
1331 cep->cm_id = id;
1332 id->add_ref(id);
1333
1334 if (!id->provider_data) {
1335 id->provider_data =
1336 kmalloc(sizeof(struct list_head), GFP_KERNEL);
1337 if (!id->provider_data) {
1338 ret = -ENOMEM;
1339 goto error;
1340 }
1341 INIT_LIST_HEAD((struct list_head *)id->provider_data);
1342 }
1343
1344 list_add_tail(&cep->listenq, (struct list_head *)id->provider_data);
1345 cep->state = ERDMA_EPSTATE_LISTENING;
1346
1347 return 0;
1348
1349 error:
1350 if (cep) {
1351 erdma_cep_set_inuse(cep);
1352
1353 if (cep->cm_id) {
1354 cep->cm_id->rem_ref(cep->cm_id);
1355 cep->cm_id = NULL;
1356 }
1357 cep->sock = NULL;
1358 erdma_socket_disassoc(s);
1359 cep->state = ERDMA_EPSTATE_CLOSED;
1360
1361 erdma_cep_set_free(cep);
1362 erdma_cep_put(cep);
1363 }
1364 sock_release(s);
1365
1366 return ret;
1367 }
1368
erdma_drop_listeners(struct iw_cm_id * id)1369 static void erdma_drop_listeners(struct iw_cm_id *id)
1370 {
1371 struct list_head *p, *tmp;
1372 /*
1373 * In case of a wildcard rdma_listen on a multi-homed device,
1374 * a listener's IWCM id is associated with more than one listening CEP.
1375 */
1376 list_for_each_safe(p, tmp, (struct list_head *)id->provider_data) {
1377 struct erdma_cep *cep =
1378 list_entry(p, struct erdma_cep, listenq);
1379
1380 list_del(p);
1381
1382 erdma_cep_set_inuse(cep);
1383
1384 if (cep->cm_id) {
1385 cep->cm_id->rem_ref(cep->cm_id);
1386 cep->cm_id = NULL;
1387 }
1388 if (cep->sock) {
1389 erdma_socket_disassoc(cep->sock);
1390 sock_release(cep->sock);
1391 cep->sock = NULL;
1392 }
1393 cep->state = ERDMA_EPSTATE_CLOSED;
1394 erdma_cep_set_free(cep);
1395 erdma_cep_put(cep);
1396 }
1397 }
1398
erdma_destroy_listen(struct iw_cm_id * id)1399 int erdma_destroy_listen(struct iw_cm_id *id)
1400 {
1401 if (!id->provider_data)
1402 return 0;
1403
1404 erdma_drop_listeners(id);
1405 kfree(id->provider_data);
1406 id->provider_data = NULL;
1407
1408 return 0;
1409 }
1410
erdma_cm_init(void)1411 int erdma_cm_init(void)
1412 {
1413 erdma_cm_wq = create_singlethread_workqueue("erdma_cm_wq");
1414 if (!erdma_cm_wq)
1415 return -ENOMEM;
1416
1417 return 0;
1418 }
1419
erdma_cm_exit(void)1420 void erdma_cm_exit(void)
1421 {
1422 if (erdma_cm_wq)
1423 destroy_workqueue(erdma_cm_wq);
1424 }
1425