1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * rio_cm - RapidIO Channelized Messaging Driver
4 *
5 * Copyright 2013-2016 Integrated Device Technology, Inc.
6 * Copyright (c) 2015, Prodrive Technologies
7 * Copyright (c) 2015, RapidIO Trade Association
8 */
9
10 #include <linux/module.h>
11 #include <linux/kernel.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/delay.h>
14 #include <linux/sched.h>
15 #include <linux/rio.h>
16 #include <linux/rio_drv.h>
17 #include <linux/slab.h>
18 #include <linux/idr.h>
19 #include <linux/interrupt.h>
20 #include <linux/cdev.h>
21 #include <linux/fs.h>
22 #include <linux/poll.h>
23 #include <linux/reboot.h>
24 #include <linux/bitops.h>
25 #include <linux/printk.h>
26 #include <linux/rio_cm_cdev.h>
27
28 #define DRV_NAME "rio_cm"
29 #define DRV_VERSION "1.0.0"
30 #define DRV_AUTHOR "Alexandre Bounine <alexandre.bounine@idt.com>"
31 #define DRV_DESC "RapidIO Channelized Messaging Driver"
32 #define DEV_NAME "rio_cm"
33
34 /* Debug output filtering masks */
35 enum {
36 DBG_NONE = 0,
37 DBG_INIT = BIT(0), /* driver init */
38 DBG_EXIT = BIT(1), /* driver exit */
39 DBG_MPORT = BIT(2), /* mport add/remove */
40 DBG_RDEV = BIT(3), /* RapidIO device add/remove */
41 DBG_CHOP = BIT(4), /* channel operations */
42 DBG_WAIT = BIT(5), /* waiting for events */
43 DBG_TX = BIT(6), /* message TX */
44 DBG_TX_EVENT = BIT(7), /* message TX event */
45 DBG_RX_DATA = BIT(8), /* inbound data messages */
46 DBG_RX_CMD = BIT(9), /* inbound REQ/ACK/NACK messages */
47 DBG_ALL = ~0,
48 };
49
50 #ifdef DEBUG
51 #define riocm_debug(level, fmt, arg...) \
52 do { \
53 if (DBG_##level & dbg_level) \
54 pr_debug(DRV_NAME ": %s " fmt "\n", \
55 __func__, ##arg); \
56 } while (0)
57 #else
58 #define riocm_debug(level, fmt, arg...) \
59 no_printk(KERN_DEBUG pr_fmt(DRV_NAME fmt "\n"), ##arg)
60 #endif
61
62 #define riocm_warn(fmt, arg...) \
63 pr_warn(DRV_NAME ": %s WARNING " fmt "\n", __func__, ##arg)
64
65 #define riocm_error(fmt, arg...) \
66 pr_err(DRV_NAME ": %s ERROR " fmt "\n", __func__, ##arg)
67
68
69 static int cmbox = 1;
70 module_param(cmbox, int, S_IRUGO);
71 MODULE_PARM_DESC(cmbox, "RapidIO Mailbox number (default 1)");
72
73 static int chstart = 256;
74 module_param(chstart, int, S_IRUGO);
75 MODULE_PARM_DESC(chstart,
76 "Start channel number for dynamic allocation (default 256)");
77
78 #ifdef DEBUG
79 static u32 dbg_level = DBG_NONE;
80 module_param(dbg_level, uint, S_IWUSR | S_IRUGO);
81 MODULE_PARM_DESC(dbg_level, "Debugging output level (default 0 = none)");
82 #endif
83
84 MODULE_AUTHOR(DRV_AUTHOR);
85 MODULE_DESCRIPTION(DRV_DESC);
86 MODULE_LICENSE("GPL");
87 MODULE_VERSION(DRV_VERSION);
88
89 #define RIOCM_TX_RING_SIZE 128
90 #define RIOCM_RX_RING_SIZE 128
91 #define RIOCM_CONNECT_TO 3 /* connect response TO (in sec) */
92
93 #define RIOCM_MAX_CHNUM 0xffff /* Use full range of u16 field */
94 #define RIOCM_CHNUM_AUTO 0
95 #define RIOCM_MAX_EP_COUNT 0x10000 /* Max number of endpoints */
96
97 enum rio_cm_state {
98 RIO_CM_IDLE,
99 RIO_CM_CONNECT,
100 RIO_CM_CONNECTED,
101 RIO_CM_DISCONNECT,
102 RIO_CM_CHAN_BOUND,
103 RIO_CM_LISTEN,
104 RIO_CM_DESTROYING,
105 };
106
107 enum rio_cm_pkt_type {
108 RIO_CM_SYS = 0xaa,
109 RIO_CM_CHAN = 0x55,
110 };
111
112 enum rio_cm_chop {
113 CM_CONN_REQ,
114 CM_CONN_ACK,
115 CM_CONN_CLOSE,
116 CM_DATA_MSG,
117 };
118
119 struct rio_ch_base_bhdr {
120 u32 src_id;
121 u32 dst_id;
122 #define RIO_HDR_LETTER_MASK 0xffff0000
123 #define RIO_HDR_MBOX_MASK 0x0000ffff
124 u8 src_mbox;
125 u8 dst_mbox;
126 u8 type;
127 } __attribute__((__packed__));
128
129 struct rio_ch_chan_hdr {
130 struct rio_ch_base_bhdr bhdr;
131 u8 ch_op;
132 u16 dst_ch;
133 u16 src_ch;
134 u16 msg_len;
135 u16 rsrvd;
136 } __attribute__((__packed__));
137
138 struct tx_req {
139 struct list_head node;
140 struct rio_dev *rdev;
141 void *buffer;
142 size_t len;
143 };
144
145 struct cm_dev {
146 struct list_head list;
147 struct rio_mport *mport;
148 void *rx_buf[RIOCM_RX_RING_SIZE];
149 int rx_slots;
150 struct mutex rx_lock;
151
152 void *tx_buf[RIOCM_TX_RING_SIZE];
153 int tx_slot;
154 int tx_cnt;
155 int tx_ack_slot;
156 struct list_head tx_reqs;
157 spinlock_t tx_lock;
158
159 struct list_head peers;
160 u32 npeers;
161 struct workqueue_struct *rx_wq;
162 struct work_struct rx_work;
163 };
164
165 struct chan_rx_ring {
166 void *buf[RIOCM_RX_RING_SIZE];
167 int head;
168 int tail;
169 int count;
170
171 /* Tracking RX buffers reported to upper level */
172 void *inuse[RIOCM_RX_RING_SIZE];
173 int inuse_cnt;
174 };
175
176 struct rio_channel {
177 u16 id; /* local channel ID */
178 struct kref ref; /* channel refcount */
179 struct file *filp;
180 struct cm_dev *cmdev; /* associated CM device object */
181 struct rio_dev *rdev; /* remote RapidIO device */
182 enum rio_cm_state state;
183 int error;
184 spinlock_t lock;
185 void *context;
186 u32 loc_destid; /* local destID */
187 u32 rem_destid; /* remote destID */
188 u16 rem_channel; /* remote channel ID */
189 struct list_head accept_queue;
190 struct list_head ch_node;
191 struct completion comp;
192 struct completion comp_close;
193 struct chan_rx_ring rx_ring;
194 };
195
196 struct cm_peer {
197 struct list_head node;
198 struct rio_dev *rdev;
199 };
200
201 struct rio_cm_work {
202 struct work_struct work;
203 struct cm_dev *cm;
204 void *data;
205 };
206
207 struct conn_req {
208 struct list_head node;
209 u32 destid; /* requester destID */
210 u16 chan; /* requester channel ID */
211 struct cm_dev *cmdev;
212 };
213
214 /*
215 * A channel_dev structure represents a CM_CDEV
216 * @cdev Character device
217 * @dev Associated device object
218 */
219 struct channel_dev {
220 struct cdev cdev;
221 struct device *dev;
222 };
223
224 static struct rio_channel *riocm_ch_alloc(u16 ch_num);
225 static void riocm_ch_free(struct kref *ref);
226 static int riocm_post_send(struct cm_dev *cm, struct rio_dev *rdev,
227 void *buffer, size_t len);
228 static int riocm_ch_close(struct rio_channel *ch);
229
230 static DEFINE_SPINLOCK(idr_lock);
231 static DEFINE_IDR(ch_idr);
232
233 static LIST_HEAD(cm_dev_list);
234 static DECLARE_RWSEM(rdev_sem);
235
236 static struct class *dev_class;
237 static unsigned int dev_major;
238 static unsigned int dev_minor_base;
239 static dev_t dev_number;
240 static struct channel_dev riocm_cdev;
241
242 #define is_msg_capable(src_ops, dst_ops) \
243 ((src_ops & RIO_SRC_OPS_DATA_MSG) && \
244 (dst_ops & RIO_DST_OPS_DATA_MSG))
245 #define dev_cm_capable(dev) \
246 is_msg_capable(dev->src_ops, dev->dst_ops)
247
riocm_cmp(struct rio_channel * ch,enum rio_cm_state cmp)248 static int riocm_cmp(struct rio_channel *ch, enum rio_cm_state cmp)
249 {
250 int ret;
251
252 spin_lock_bh(&ch->lock);
253 ret = (ch->state == cmp);
254 spin_unlock_bh(&ch->lock);
255 return ret;
256 }
257
riocm_cmp_exch(struct rio_channel * ch,enum rio_cm_state cmp,enum rio_cm_state exch)258 static int riocm_cmp_exch(struct rio_channel *ch,
259 enum rio_cm_state cmp, enum rio_cm_state exch)
260 {
261 int ret;
262
263 spin_lock_bh(&ch->lock);
264 ret = (ch->state == cmp);
265 if (ret)
266 ch->state = exch;
267 spin_unlock_bh(&ch->lock);
268 return ret;
269 }
270
riocm_exch(struct rio_channel * ch,enum rio_cm_state exch)271 static enum rio_cm_state riocm_exch(struct rio_channel *ch,
272 enum rio_cm_state exch)
273 {
274 enum rio_cm_state old;
275
276 spin_lock_bh(&ch->lock);
277 old = ch->state;
278 ch->state = exch;
279 spin_unlock_bh(&ch->lock);
280 return old;
281 }
282
riocm_get_channel(u16 nr)283 static struct rio_channel *riocm_get_channel(u16 nr)
284 {
285 struct rio_channel *ch;
286
287 spin_lock_bh(&idr_lock);
288 ch = idr_find(&ch_idr, nr);
289 if (ch)
290 kref_get(&ch->ref);
291 spin_unlock_bh(&idr_lock);
292 return ch;
293 }
294
riocm_put_channel(struct rio_channel * ch)295 static void riocm_put_channel(struct rio_channel *ch)
296 {
297 kref_put(&ch->ref, riocm_ch_free);
298 }
299
riocm_rx_get_msg(struct cm_dev * cm)300 static void *riocm_rx_get_msg(struct cm_dev *cm)
301 {
302 void *msg;
303 int i;
304
305 msg = rio_get_inb_message(cm->mport, cmbox);
306 if (msg) {
307 for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
308 if (cm->rx_buf[i] == msg) {
309 cm->rx_buf[i] = NULL;
310 cm->rx_slots++;
311 break;
312 }
313 }
314
315 if (i == RIOCM_RX_RING_SIZE)
316 riocm_warn("no record for buffer 0x%p", msg);
317 }
318
319 return msg;
320 }
321
322 /*
323 * riocm_rx_fill - fills a ring of receive buffers for given cm device
324 * @cm: cm_dev object
325 * @nent: max number of entries to fill
326 *
327 * Returns: none
328 */
riocm_rx_fill(struct cm_dev * cm,int nent)329 static void riocm_rx_fill(struct cm_dev *cm, int nent)
330 {
331 int i;
332
333 if (cm->rx_slots == 0)
334 return;
335
336 for (i = 0; i < RIOCM_RX_RING_SIZE && cm->rx_slots && nent; i++) {
337 if (cm->rx_buf[i] == NULL) {
338 cm->rx_buf[i] = kmalloc(RIO_MAX_MSG_SIZE, GFP_KERNEL);
339 if (cm->rx_buf[i] == NULL)
340 break;
341 rio_add_inb_buffer(cm->mport, cmbox, cm->rx_buf[i]);
342 cm->rx_slots--;
343 nent--;
344 }
345 }
346 }
347
348 /*
349 * riocm_rx_free - frees all receive buffers associated with given cm device
350 * @cm: cm_dev object
351 *
352 * Returns: none
353 */
riocm_rx_free(struct cm_dev * cm)354 static void riocm_rx_free(struct cm_dev *cm)
355 {
356 int i;
357
358 for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
359 if (cm->rx_buf[i] != NULL) {
360 kfree(cm->rx_buf[i]);
361 cm->rx_buf[i] = NULL;
362 }
363 }
364 }
365
366 /*
367 * riocm_req_handler - connection request handler
368 * @cm: cm_dev object
369 * @req_data: pointer to the request packet
370 *
371 * Returns: 0 if success, or
372 * -EINVAL if channel is not in correct state,
373 * -ENODEV if cannot find a channel with specified ID,
374 * -ENOMEM if unable to allocate memory to store the request
375 */
riocm_req_handler(struct cm_dev * cm,void * req_data)376 static int riocm_req_handler(struct cm_dev *cm, void *req_data)
377 {
378 struct rio_channel *ch;
379 struct conn_req *req;
380 struct rio_ch_chan_hdr *hh = req_data;
381 u16 chnum;
382
383 chnum = ntohs(hh->dst_ch);
384
385 ch = riocm_get_channel(chnum);
386
387 if (!ch)
388 return -ENODEV;
389
390 if (ch->state != RIO_CM_LISTEN) {
391 riocm_debug(RX_CMD, "channel %d is not in listen state", chnum);
392 riocm_put_channel(ch);
393 return -EINVAL;
394 }
395
396 req = kzalloc(sizeof(*req), GFP_KERNEL);
397 if (!req) {
398 riocm_put_channel(ch);
399 return -ENOMEM;
400 }
401
402 req->destid = ntohl(hh->bhdr.src_id);
403 req->chan = ntohs(hh->src_ch);
404 req->cmdev = cm;
405
406 spin_lock_bh(&ch->lock);
407 list_add_tail(&req->node, &ch->accept_queue);
408 spin_unlock_bh(&ch->lock);
409 complete(&ch->comp);
410 riocm_put_channel(ch);
411
412 return 0;
413 }
414
415 /*
416 * riocm_resp_handler - response to connection request handler
417 * @resp_data: pointer to the response packet
418 *
419 * Returns: 0 if success, or
420 * -EINVAL if channel is not in correct state,
421 * -ENODEV if cannot find a channel with specified ID,
422 */
riocm_resp_handler(void * resp_data)423 static int riocm_resp_handler(void *resp_data)
424 {
425 struct rio_channel *ch;
426 struct rio_ch_chan_hdr *hh = resp_data;
427 u16 chnum;
428
429 chnum = ntohs(hh->dst_ch);
430 ch = riocm_get_channel(chnum);
431 if (!ch)
432 return -ENODEV;
433
434 if (ch->state != RIO_CM_CONNECT) {
435 riocm_put_channel(ch);
436 return -EINVAL;
437 }
438
439 riocm_exch(ch, RIO_CM_CONNECTED);
440 ch->rem_channel = ntohs(hh->src_ch);
441 complete(&ch->comp);
442 riocm_put_channel(ch);
443
444 return 0;
445 }
446
447 /*
448 * riocm_close_handler - channel close request handler
449 * @req_data: pointer to the request packet
450 *
451 * Returns: 0 if success, or
452 * -ENODEV if cannot find a channel with specified ID,
453 * + error codes returned by riocm_ch_close.
454 */
riocm_close_handler(void * data)455 static int riocm_close_handler(void *data)
456 {
457 struct rio_channel *ch;
458 struct rio_ch_chan_hdr *hh = data;
459 int ret;
460
461 riocm_debug(RX_CMD, "for ch=%d", ntohs(hh->dst_ch));
462
463 spin_lock_bh(&idr_lock);
464 ch = idr_find(&ch_idr, ntohs(hh->dst_ch));
465 if (!ch) {
466 spin_unlock_bh(&idr_lock);
467 return -ENODEV;
468 }
469 idr_remove(&ch_idr, ch->id);
470 spin_unlock_bh(&idr_lock);
471
472 riocm_exch(ch, RIO_CM_DISCONNECT);
473
474 ret = riocm_ch_close(ch);
475 if (ret)
476 riocm_debug(RX_CMD, "riocm_ch_close() returned %d", ret);
477
478 return 0;
479 }
480
481 /*
482 * rio_cm_handler - function that services request (non-data) packets
483 * @cm: cm_dev object
484 * @data: pointer to the packet
485 */
rio_cm_handler(struct cm_dev * cm,void * data)486 static void rio_cm_handler(struct cm_dev *cm, void *data)
487 {
488 struct rio_ch_chan_hdr *hdr;
489
490 if (!rio_mport_is_running(cm->mport))
491 goto out;
492
493 hdr = data;
494
495 riocm_debug(RX_CMD, "OP=%x for ch=%d from %d",
496 hdr->ch_op, ntohs(hdr->dst_ch), ntohs(hdr->src_ch));
497
498 switch (hdr->ch_op) {
499 case CM_CONN_REQ:
500 riocm_req_handler(cm, data);
501 break;
502 case CM_CONN_ACK:
503 riocm_resp_handler(data);
504 break;
505 case CM_CONN_CLOSE:
506 riocm_close_handler(data);
507 break;
508 default:
509 riocm_error("Invalid packet header");
510 break;
511 }
512 out:
513 kfree(data);
514 }
515
516 /*
517 * rio_rx_data_handler - received data packet handler
518 * @cm: cm_dev object
519 * @buf: data packet
520 *
521 * Returns: 0 if success, or
522 * -ENODEV if cannot find a channel with specified ID,
523 * -EIO if channel is not in CONNECTED state,
524 * -ENOMEM if channel RX queue is full (packet discarded)
525 */
rio_rx_data_handler(struct cm_dev * cm,void * buf)526 static int rio_rx_data_handler(struct cm_dev *cm, void *buf)
527 {
528 struct rio_ch_chan_hdr *hdr;
529 struct rio_channel *ch;
530
531 hdr = buf;
532
533 riocm_debug(RX_DATA, "for ch=%d", ntohs(hdr->dst_ch));
534
535 ch = riocm_get_channel(ntohs(hdr->dst_ch));
536 if (!ch) {
537 /* Discard data message for non-existing channel */
538 kfree(buf);
539 return -ENODEV;
540 }
541
542 /* Place pointer to the buffer into channel's RX queue */
543 spin_lock(&ch->lock);
544
545 if (ch->state != RIO_CM_CONNECTED) {
546 /* Channel is not ready to receive data, discard a packet */
547 riocm_debug(RX_DATA, "ch=%d is in wrong state=%d",
548 ch->id, ch->state);
549 spin_unlock(&ch->lock);
550 kfree(buf);
551 riocm_put_channel(ch);
552 return -EIO;
553 }
554
555 if (ch->rx_ring.count == RIOCM_RX_RING_SIZE) {
556 /* If RX ring is full, discard a packet */
557 riocm_debug(RX_DATA, "ch=%d is full", ch->id);
558 spin_unlock(&ch->lock);
559 kfree(buf);
560 riocm_put_channel(ch);
561 return -ENOMEM;
562 }
563
564 ch->rx_ring.buf[ch->rx_ring.head] = buf;
565 ch->rx_ring.head++;
566 ch->rx_ring.count++;
567 ch->rx_ring.head %= RIOCM_RX_RING_SIZE;
568
569 complete(&ch->comp);
570
571 spin_unlock(&ch->lock);
572 riocm_put_channel(ch);
573
574 return 0;
575 }
576
577 /*
578 * rio_ibmsg_handler - inbound message packet handler
579 */
rio_ibmsg_handler(struct work_struct * work)580 static void rio_ibmsg_handler(struct work_struct *work)
581 {
582 struct cm_dev *cm = container_of(work, struct cm_dev, rx_work);
583 void *data;
584 struct rio_ch_chan_hdr *hdr;
585
586 if (!rio_mport_is_running(cm->mport))
587 return;
588
589 while (1) {
590 mutex_lock(&cm->rx_lock);
591 data = riocm_rx_get_msg(cm);
592 if (data)
593 riocm_rx_fill(cm, 1);
594 mutex_unlock(&cm->rx_lock);
595
596 if (data == NULL)
597 break;
598
599 hdr = data;
600
601 if (hdr->bhdr.type != RIO_CM_CHAN) {
602 /* For now simply discard packets other than channel */
603 riocm_error("Unsupported TYPE code (0x%x). Msg dropped",
604 hdr->bhdr.type);
605 kfree(data);
606 continue;
607 }
608
609 /* Process a channel message */
610 if (hdr->ch_op == CM_DATA_MSG)
611 rio_rx_data_handler(cm, data);
612 else
613 rio_cm_handler(cm, data);
614 }
615 }
616
riocm_inb_msg_event(struct rio_mport * mport,void * dev_id,int mbox,int slot)617 static void riocm_inb_msg_event(struct rio_mport *mport, void *dev_id,
618 int mbox, int slot)
619 {
620 struct cm_dev *cm = dev_id;
621
622 if (rio_mport_is_running(cm->mport) && !work_pending(&cm->rx_work))
623 queue_work(cm->rx_wq, &cm->rx_work);
624 }
625
626 /*
627 * rio_txcq_handler - TX completion handler
628 * @cm: cm_dev object
629 * @slot: TX queue slot
630 *
631 * TX completion handler also ensures that pending request packets are placed
632 * into transmit queue as soon as a free slot becomes available. This is done
633 * to give higher priority to request packets during high intensity data flow.
634 */
rio_txcq_handler(struct cm_dev * cm,int slot)635 static void rio_txcq_handler(struct cm_dev *cm, int slot)
636 {
637 int ack_slot;
638
639 /* ATTN: Add TX completion notification if/when direct buffer
640 * transfer is implemented. At this moment only correct tracking
641 * of tx_count is important.
642 */
643 riocm_debug(TX_EVENT, "for mport_%d slot %d tx_cnt %d",
644 cm->mport->id, slot, cm->tx_cnt);
645
646 spin_lock(&cm->tx_lock);
647 ack_slot = cm->tx_ack_slot;
648
649 if (ack_slot == slot)
650 riocm_debug(TX_EVENT, "slot == ack_slot");
651
652 while (cm->tx_cnt && ((ack_slot != slot) ||
653 (cm->tx_cnt == RIOCM_TX_RING_SIZE))) {
654
655 cm->tx_buf[ack_slot] = NULL;
656 ++ack_slot;
657 ack_slot &= (RIOCM_TX_RING_SIZE - 1);
658 cm->tx_cnt--;
659 }
660
661 if (cm->tx_cnt < 0 || cm->tx_cnt > RIOCM_TX_RING_SIZE)
662 riocm_error("tx_cnt %d out of sync", cm->tx_cnt);
663
664 WARN_ON((cm->tx_cnt < 0) || (cm->tx_cnt > RIOCM_TX_RING_SIZE));
665
666 cm->tx_ack_slot = ack_slot;
667
668 /*
669 * If there are pending requests, insert them into transmit queue
670 */
671 if (!list_empty(&cm->tx_reqs) && (cm->tx_cnt < RIOCM_TX_RING_SIZE)) {
672 struct tx_req *req, *_req;
673 int rc;
674
675 list_for_each_entry_safe(req, _req, &cm->tx_reqs, node) {
676 list_del(&req->node);
677 cm->tx_buf[cm->tx_slot] = req->buffer;
678 rc = rio_add_outb_message(cm->mport, req->rdev, cmbox,
679 req->buffer, req->len);
680 kfree(req->buffer);
681 kfree(req);
682
683 ++cm->tx_cnt;
684 ++cm->tx_slot;
685 cm->tx_slot &= (RIOCM_TX_RING_SIZE - 1);
686 if (cm->tx_cnt == RIOCM_TX_RING_SIZE)
687 break;
688 }
689 }
690
691 spin_unlock(&cm->tx_lock);
692 }
693
riocm_outb_msg_event(struct rio_mport * mport,void * dev_id,int mbox,int slot)694 static void riocm_outb_msg_event(struct rio_mport *mport, void *dev_id,
695 int mbox, int slot)
696 {
697 struct cm_dev *cm = dev_id;
698
699 if (cm && rio_mport_is_running(cm->mport))
700 rio_txcq_handler(cm, slot);
701 }
702
riocm_queue_req(struct cm_dev * cm,struct rio_dev * rdev,void * buffer,size_t len)703 static int riocm_queue_req(struct cm_dev *cm, struct rio_dev *rdev,
704 void *buffer, size_t len)
705 {
706 unsigned long flags;
707 struct tx_req *treq;
708
709 treq = kzalloc(sizeof(*treq), GFP_KERNEL);
710 if (treq == NULL)
711 return -ENOMEM;
712
713 treq->rdev = rdev;
714 treq->buffer = buffer;
715 treq->len = len;
716
717 spin_lock_irqsave(&cm->tx_lock, flags);
718 list_add_tail(&treq->node, &cm->tx_reqs);
719 spin_unlock_irqrestore(&cm->tx_lock, flags);
720 return 0;
721 }
722
723 /*
724 * riocm_post_send - helper function that places packet into msg TX queue
725 * @cm: cm_dev object
726 * @rdev: target RapidIO device object (required by outbound msg interface)
727 * @buffer: pointer to a packet buffer to send
728 * @len: length of data to transfer
729 * @req: request priority flag
730 *
731 * Returns: 0 if success, or error code otherwise.
732 */
riocm_post_send(struct cm_dev * cm,struct rio_dev * rdev,void * buffer,size_t len)733 static int riocm_post_send(struct cm_dev *cm, struct rio_dev *rdev,
734 void *buffer, size_t len)
735 {
736 int rc;
737 unsigned long flags;
738
739 spin_lock_irqsave(&cm->tx_lock, flags);
740
741 if (cm->mport == NULL) {
742 rc = -ENODEV;
743 goto err_out;
744 }
745
746 if (cm->tx_cnt == RIOCM_TX_RING_SIZE) {
747 riocm_debug(TX, "Tx Queue is full");
748 rc = -EBUSY;
749 goto err_out;
750 }
751
752 cm->tx_buf[cm->tx_slot] = buffer;
753 rc = rio_add_outb_message(cm->mport, rdev, cmbox, buffer, len);
754
755 riocm_debug(TX, "Add buf@%p destid=%x tx_slot=%d tx_cnt=%d",
756 buffer, rdev->destid, cm->tx_slot, cm->tx_cnt);
757
758 ++cm->tx_cnt;
759 ++cm->tx_slot;
760 cm->tx_slot &= (RIOCM_TX_RING_SIZE - 1);
761
762 err_out:
763 spin_unlock_irqrestore(&cm->tx_lock, flags);
764 return rc;
765 }
766
767 /*
768 * riocm_ch_send - sends a data packet to a remote device
769 * @ch_id: local channel ID
770 * @buf: pointer to a data buffer to send (including CM header)
771 * @len: length of data to transfer (including CM header)
772 *
773 * ATTN: ASSUMES THAT THE HEADER SPACE IS RESERVED PART OF THE DATA PACKET
774 *
775 * Returns: 0 if success, or
776 * -EINVAL if one or more input parameters is/are not valid,
777 * -ENODEV if cannot find a channel with specified ID,
778 * -EAGAIN if a channel is not in CONNECTED state,
779 * + error codes returned by HW send routine.
780 */
riocm_ch_send(u16 ch_id,void * buf,int len)781 static int riocm_ch_send(u16 ch_id, void *buf, int len)
782 {
783 struct rio_channel *ch;
784 struct rio_ch_chan_hdr *hdr;
785 int ret;
786
787 if (buf == NULL || ch_id == 0 || len == 0 || len > RIO_MAX_MSG_SIZE)
788 return -EINVAL;
789
790 if (len < sizeof(struct rio_ch_chan_hdr))
791 return -EINVAL; /* insufficient data from user */
792
793 ch = riocm_get_channel(ch_id);
794 if (!ch) {
795 riocm_error("%s(%d) ch_%d not found", current->comm,
796 task_pid_nr(current), ch_id);
797 return -ENODEV;
798 }
799
800 if (!riocm_cmp(ch, RIO_CM_CONNECTED)) {
801 ret = -EAGAIN;
802 goto err_out;
803 }
804
805 /*
806 * Fill buffer header section with corresponding channel data
807 */
808 hdr = buf;
809
810 hdr->bhdr.src_id = htonl(ch->loc_destid);
811 hdr->bhdr.dst_id = htonl(ch->rem_destid);
812 hdr->bhdr.src_mbox = cmbox;
813 hdr->bhdr.dst_mbox = cmbox;
814 hdr->bhdr.type = RIO_CM_CHAN;
815 hdr->ch_op = CM_DATA_MSG;
816 hdr->dst_ch = htons(ch->rem_channel);
817 hdr->src_ch = htons(ch->id);
818 hdr->msg_len = htons((u16)len);
819
820 /* ATTN: the function call below relies on the fact that underlying
821 * HW-specific add_outb_message() routine copies TX data into its own
822 * internal transfer buffer (true for all RIONET compatible mport
823 * drivers). Must be reviewed if mport driver uses the buffer directly.
824 */
825
826 ret = riocm_post_send(ch->cmdev, ch->rdev, buf, len);
827 if (ret)
828 riocm_debug(TX, "ch %d send_err=%d", ch->id, ret);
829 err_out:
830 riocm_put_channel(ch);
831 return ret;
832 }
833
riocm_ch_free_rxbuf(struct rio_channel * ch,void * buf)834 static int riocm_ch_free_rxbuf(struct rio_channel *ch, void *buf)
835 {
836 int i, ret = -EINVAL;
837
838 spin_lock_bh(&ch->lock);
839
840 for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
841 if (ch->rx_ring.inuse[i] == buf) {
842 ch->rx_ring.inuse[i] = NULL;
843 ch->rx_ring.inuse_cnt--;
844 ret = 0;
845 break;
846 }
847 }
848
849 spin_unlock_bh(&ch->lock);
850
851 if (!ret)
852 kfree(buf);
853
854 return ret;
855 }
856
857 /*
858 * riocm_ch_receive - fetch a data packet received for the specified channel
859 * @ch: local channel ID
860 * @buf: pointer to a packet buffer
861 * @timeout: timeout to wait for incoming packet (in jiffies)
862 *
863 * Returns: 0 and valid buffer pointer if success, or NULL pointer and one of:
864 * -EAGAIN if a channel is not in CONNECTED state,
865 * -ENOMEM if in-use tracking queue is full,
866 * -ETIME if wait timeout expired,
867 * -EINTR if wait was interrupted.
868 */
riocm_ch_receive(struct rio_channel * ch,void ** buf,long timeout)869 static int riocm_ch_receive(struct rio_channel *ch, void **buf, long timeout)
870 {
871 void *rxmsg = NULL;
872 int i, ret = 0;
873 long wret;
874
875 if (!riocm_cmp(ch, RIO_CM_CONNECTED)) {
876 ret = -EAGAIN;
877 goto out;
878 }
879
880 if (ch->rx_ring.inuse_cnt == RIOCM_RX_RING_SIZE) {
881 /* If we do not have entries to track buffers given to upper
882 * layer, reject request.
883 */
884 ret = -ENOMEM;
885 goto out;
886 }
887
888 wret = wait_for_completion_interruptible_timeout(&ch->comp, timeout);
889
890 riocm_debug(WAIT, "wait on %d returned %ld", ch->id, wret);
891
892 if (!wret)
893 ret = -ETIME;
894 else if (wret == -ERESTARTSYS)
895 ret = -EINTR;
896 else
897 ret = riocm_cmp(ch, RIO_CM_CONNECTED) ? 0 : -ECONNRESET;
898
899 if (ret)
900 goto out;
901
902 spin_lock_bh(&ch->lock);
903
904 rxmsg = ch->rx_ring.buf[ch->rx_ring.tail];
905 ch->rx_ring.buf[ch->rx_ring.tail] = NULL;
906 ch->rx_ring.count--;
907 ch->rx_ring.tail++;
908 ch->rx_ring.tail %= RIOCM_RX_RING_SIZE;
909 ret = -ENOMEM;
910
911 for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
912 if (ch->rx_ring.inuse[i] == NULL) {
913 ch->rx_ring.inuse[i] = rxmsg;
914 ch->rx_ring.inuse_cnt++;
915 ret = 0;
916 break;
917 }
918 }
919
920 if (ret) {
921 /* We have no entry to store pending message: drop it */
922 kfree(rxmsg);
923 rxmsg = NULL;
924 }
925
926 spin_unlock_bh(&ch->lock);
927 out:
928 *buf = rxmsg;
929 return ret;
930 }
931
932 /*
933 * riocm_ch_connect - sends a connect request to a remote device
934 * @loc_ch: local channel ID
935 * @cm: CM device to send connect request
936 * @peer: target RapidIO device
937 * @rem_ch: remote channel ID
938 *
939 * Returns: 0 if success, or
940 * -EINVAL if the channel is not in IDLE state,
941 * -EAGAIN if no connection request available immediately,
942 * -ETIME if ACK response timeout expired,
943 * -EINTR if wait for response was interrupted.
944 */
riocm_ch_connect(u16 loc_ch,struct cm_dev * cm,struct cm_peer * peer,u16 rem_ch)945 static int riocm_ch_connect(u16 loc_ch, struct cm_dev *cm,
946 struct cm_peer *peer, u16 rem_ch)
947 {
948 struct rio_channel *ch = NULL;
949 struct rio_ch_chan_hdr *hdr;
950 int ret;
951 long wret;
952
953 ch = riocm_get_channel(loc_ch);
954 if (!ch)
955 return -ENODEV;
956
957 if (!riocm_cmp_exch(ch, RIO_CM_IDLE, RIO_CM_CONNECT)) {
958 ret = -EINVAL;
959 goto conn_done;
960 }
961
962 ch->cmdev = cm;
963 ch->rdev = peer->rdev;
964 ch->context = NULL;
965 ch->loc_destid = cm->mport->host_deviceid;
966 ch->rem_channel = rem_ch;
967
968 /*
969 * Send connect request to the remote RapidIO device
970 */
971
972 hdr = kzalloc(sizeof(*hdr), GFP_KERNEL);
973 if (hdr == NULL) {
974 ret = -ENOMEM;
975 goto conn_done;
976 }
977
978 hdr->bhdr.src_id = htonl(ch->loc_destid);
979 hdr->bhdr.dst_id = htonl(peer->rdev->destid);
980 hdr->bhdr.src_mbox = cmbox;
981 hdr->bhdr.dst_mbox = cmbox;
982 hdr->bhdr.type = RIO_CM_CHAN;
983 hdr->ch_op = CM_CONN_REQ;
984 hdr->dst_ch = htons(rem_ch);
985 hdr->src_ch = htons(loc_ch);
986
987 /* ATTN: the function call below relies on the fact that underlying
988 * HW-specific add_outb_message() routine copies TX data into its
989 * internal transfer buffer. Must be reviewed if mport driver uses
990 * this buffer directly.
991 */
992 ret = riocm_post_send(cm, peer->rdev, hdr, sizeof(*hdr));
993
994 if (ret != -EBUSY) {
995 kfree(hdr);
996 } else {
997 ret = riocm_queue_req(cm, peer->rdev, hdr, sizeof(*hdr));
998 if (ret)
999 kfree(hdr);
1000 }
1001
1002 if (ret) {
1003 riocm_cmp_exch(ch, RIO_CM_CONNECT, RIO_CM_IDLE);
1004 goto conn_done;
1005 }
1006
1007 /* Wait for connect response from the remote device */
1008 wret = wait_for_completion_interruptible_timeout(&ch->comp,
1009 RIOCM_CONNECT_TO * HZ);
1010 riocm_debug(WAIT, "wait on %d returns %ld", ch->id, wret);
1011
1012 if (!wret)
1013 ret = -ETIME;
1014 else if (wret == -ERESTARTSYS)
1015 ret = -EINTR;
1016 else
1017 ret = riocm_cmp(ch, RIO_CM_CONNECTED) ? 0 : -1;
1018
1019 conn_done:
1020 riocm_put_channel(ch);
1021 return ret;
1022 }
1023
riocm_send_ack(struct rio_channel * ch)1024 static int riocm_send_ack(struct rio_channel *ch)
1025 {
1026 struct rio_ch_chan_hdr *hdr;
1027 int ret;
1028
1029 hdr = kzalloc(sizeof(*hdr), GFP_KERNEL);
1030 if (hdr == NULL)
1031 return -ENOMEM;
1032
1033 hdr->bhdr.src_id = htonl(ch->loc_destid);
1034 hdr->bhdr.dst_id = htonl(ch->rem_destid);
1035 hdr->dst_ch = htons(ch->rem_channel);
1036 hdr->src_ch = htons(ch->id);
1037 hdr->bhdr.src_mbox = cmbox;
1038 hdr->bhdr.dst_mbox = cmbox;
1039 hdr->bhdr.type = RIO_CM_CHAN;
1040 hdr->ch_op = CM_CONN_ACK;
1041
1042 /* ATTN: the function call below relies on the fact that underlying
1043 * add_outb_message() routine copies TX data into its internal transfer
1044 * buffer. Review if switching to direct buffer version.
1045 */
1046 ret = riocm_post_send(ch->cmdev, ch->rdev, hdr, sizeof(*hdr));
1047
1048 if (ret == -EBUSY && !riocm_queue_req(ch->cmdev,
1049 ch->rdev, hdr, sizeof(*hdr)))
1050 return 0;
1051 kfree(hdr);
1052
1053 if (ret)
1054 riocm_error("send ACK to ch_%d on %s failed (ret=%d)",
1055 ch->id, rio_name(ch->rdev), ret);
1056 return ret;
1057 }
1058
1059 /*
1060 * riocm_ch_accept - accept incoming connection request
1061 * @ch_id: channel ID
1062 * @new_ch_id: local mport device
1063 * @timeout: wait timeout (if 0 non-blocking call, do not wait if connection
1064 * request is not available).
1065 *
1066 * Returns: pointer to new channel struct if success, or error-valued pointer:
1067 * -ENODEV - cannot find specified channel or mport,
1068 * -EINVAL - the channel is not in IDLE state,
1069 * -EAGAIN - no connection request available immediately (timeout=0),
1070 * -ENOMEM - unable to allocate new channel,
1071 * -ETIME - wait timeout expired,
1072 * -EINTR - wait was interrupted.
1073 */
riocm_ch_accept(u16 ch_id,u16 * new_ch_id,long timeout)1074 static struct rio_channel *riocm_ch_accept(u16 ch_id, u16 *new_ch_id,
1075 long timeout)
1076 {
1077 struct rio_channel *ch;
1078 struct rio_channel *new_ch;
1079 struct conn_req *req;
1080 struct cm_peer *peer;
1081 int found = 0;
1082 int err = 0;
1083 long wret;
1084
1085 ch = riocm_get_channel(ch_id);
1086 if (!ch)
1087 return ERR_PTR(-EINVAL);
1088
1089 if (!riocm_cmp(ch, RIO_CM_LISTEN)) {
1090 err = -EINVAL;
1091 goto err_put;
1092 }
1093
1094 /* Don't sleep if this is a non blocking call */
1095 if (!timeout) {
1096 if (!try_wait_for_completion(&ch->comp)) {
1097 err = -EAGAIN;
1098 goto err_put;
1099 }
1100 } else {
1101 riocm_debug(WAIT, "on %d", ch->id);
1102
1103 wret = wait_for_completion_interruptible_timeout(&ch->comp,
1104 timeout);
1105 if (!wret) {
1106 err = -ETIME;
1107 goto err_put;
1108 } else if (wret == -ERESTARTSYS) {
1109 err = -EINTR;
1110 goto err_put;
1111 }
1112 }
1113
1114 spin_lock_bh(&ch->lock);
1115
1116 if (ch->state != RIO_CM_LISTEN) {
1117 err = -ECANCELED;
1118 } else if (list_empty(&ch->accept_queue)) {
1119 riocm_debug(WAIT, "on %d accept_queue is empty on completion",
1120 ch->id);
1121 err = -EIO;
1122 }
1123
1124 spin_unlock_bh(&ch->lock);
1125
1126 if (err) {
1127 riocm_debug(WAIT, "on %d returns %d", ch->id, err);
1128 goto err_put;
1129 }
1130
1131 /* Create new channel for this connection */
1132 new_ch = riocm_ch_alloc(RIOCM_CHNUM_AUTO);
1133
1134 if (IS_ERR(new_ch)) {
1135 riocm_error("failed to get channel for new req (%ld)",
1136 PTR_ERR(new_ch));
1137 err = -ENOMEM;
1138 goto err_put;
1139 }
1140
1141 spin_lock_bh(&ch->lock);
1142
1143 req = list_first_entry(&ch->accept_queue, struct conn_req, node);
1144 list_del(&req->node);
1145 new_ch->cmdev = ch->cmdev;
1146 new_ch->loc_destid = ch->loc_destid;
1147 new_ch->rem_destid = req->destid;
1148 new_ch->rem_channel = req->chan;
1149
1150 spin_unlock_bh(&ch->lock);
1151 riocm_put_channel(ch);
1152 ch = NULL;
1153 kfree(req);
1154
1155 down_read(&rdev_sem);
1156 /* Find requester's device object */
1157 list_for_each_entry(peer, &new_ch->cmdev->peers, node) {
1158 if (peer->rdev->destid == new_ch->rem_destid) {
1159 riocm_debug(RX_CMD, "found matching device(%s)",
1160 rio_name(peer->rdev));
1161 found = 1;
1162 break;
1163 }
1164 }
1165 up_read(&rdev_sem);
1166
1167 if (!found) {
1168 /* If peer device object not found, simply ignore the request */
1169 err = -ENODEV;
1170 goto err_put_new_ch;
1171 }
1172
1173 new_ch->rdev = peer->rdev;
1174 new_ch->state = RIO_CM_CONNECTED;
1175 spin_lock_init(&new_ch->lock);
1176
1177 /* Acknowledge the connection request. */
1178 riocm_send_ack(new_ch);
1179
1180 *new_ch_id = new_ch->id;
1181 return new_ch;
1182
1183 err_put_new_ch:
1184 spin_lock_bh(&idr_lock);
1185 idr_remove(&ch_idr, new_ch->id);
1186 spin_unlock_bh(&idr_lock);
1187 riocm_put_channel(new_ch);
1188
1189 err_put:
1190 if (ch)
1191 riocm_put_channel(ch);
1192 *new_ch_id = 0;
1193 return ERR_PTR(err);
1194 }
1195
1196 /*
1197 * riocm_ch_listen - puts a channel into LISTEN state
1198 * @ch_id: channel ID
1199 *
1200 * Returns: 0 if success, or
1201 * -EINVAL if the specified channel does not exists or
1202 * is not in CHAN_BOUND state.
1203 */
riocm_ch_listen(u16 ch_id)1204 static int riocm_ch_listen(u16 ch_id)
1205 {
1206 struct rio_channel *ch = NULL;
1207 int ret = 0;
1208
1209 riocm_debug(CHOP, "(ch_%d)", ch_id);
1210
1211 ch = riocm_get_channel(ch_id);
1212 if (!ch)
1213 return -EINVAL;
1214 if (!riocm_cmp_exch(ch, RIO_CM_CHAN_BOUND, RIO_CM_LISTEN))
1215 ret = -EINVAL;
1216 riocm_put_channel(ch);
1217 return ret;
1218 }
1219
1220 /*
1221 * riocm_ch_bind - associate a channel object and an mport device
1222 * @ch_id: channel ID
1223 * @mport_id: local mport device ID
1224 * @context: pointer to the additional caller's context
1225 *
1226 * Returns: 0 if success, or
1227 * -ENODEV if cannot find specified mport,
1228 * -EINVAL if the specified channel does not exist or
1229 * is not in IDLE state.
1230 */
riocm_ch_bind(u16 ch_id,u8 mport_id,void * context)1231 static int riocm_ch_bind(u16 ch_id, u8 mport_id, void *context)
1232 {
1233 struct rio_channel *ch = NULL;
1234 struct cm_dev *cm;
1235 int rc = -ENODEV;
1236
1237 riocm_debug(CHOP, "ch_%d to mport_%d", ch_id, mport_id);
1238
1239 /* Find matching cm_dev object */
1240 down_read(&rdev_sem);
1241 list_for_each_entry(cm, &cm_dev_list, list) {
1242 if ((cm->mport->id == mport_id) &&
1243 rio_mport_is_running(cm->mport)) {
1244 rc = 0;
1245 break;
1246 }
1247 }
1248
1249 if (rc)
1250 goto exit;
1251
1252 ch = riocm_get_channel(ch_id);
1253 if (!ch) {
1254 rc = -EINVAL;
1255 goto exit;
1256 }
1257
1258 spin_lock_bh(&ch->lock);
1259 if (ch->state != RIO_CM_IDLE) {
1260 spin_unlock_bh(&ch->lock);
1261 rc = -EINVAL;
1262 goto err_put;
1263 }
1264
1265 ch->cmdev = cm;
1266 ch->loc_destid = cm->mport->host_deviceid;
1267 ch->context = context;
1268 ch->state = RIO_CM_CHAN_BOUND;
1269 spin_unlock_bh(&ch->lock);
1270 err_put:
1271 riocm_put_channel(ch);
1272 exit:
1273 up_read(&rdev_sem);
1274 return rc;
1275 }
1276
1277 /*
1278 * riocm_ch_alloc - channel object allocation helper routine
1279 * @ch_num: channel ID (1 ... RIOCM_MAX_CHNUM, 0 = automatic)
1280 *
1281 * Return value: pointer to newly created channel object,
1282 * or error-valued pointer
1283 */
riocm_ch_alloc(u16 ch_num)1284 static struct rio_channel *riocm_ch_alloc(u16 ch_num)
1285 {
1286 int id;
1287 int start, end;
1288 struct rio_channel *ch;
1289
1290 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
1291 if (!ch)
1292 return ERR_PTR(-ENOMEM);
1293
1294 if (ch_num) {
1295 /* If requested, try to obtain the specified channel ID */
1296 start = ch_num;
1297 end = ch_num + 1;
1298 } else {
1299 /* Obtain channel ID from the dynamic allocation range */
1300 start = chstart;
1301 end = RIOCM_MAX_CHNUM + 1;
1302 }
1303
1304 idr_preload(GFP_KERNEL);
1305 spin_lock_bh(&idr_lock);
1306 id = idr_alloc_cyclic(&ch_idr, ch, start, end, GFP_NOWAIT);
1307 spin_unlock_bh(&idr_lock);
1308 idr_preload_end();
1309
1310 if (id < 0) {
1311 kfree(ch);
1312 return ERR_PTR(id == -ENOSPC ? -EBUSY : id);
1313 }
1314
1315 ch->id = (u16)id;
1316 ch->state = RIO_CM_IDLE;
1317 spin_lock_init(&ch->lock);
1318 INIT_LIST_HEAD(&ch->accept_queue);
1319 INIT_LIST_HEAD(&ch->ch_node);
1320 init_completion(&ch->comp);
1321 init_completion(&ch->comp_close);
1322 kref_init(&ch->ref);
1323 ch->rx_ring.head = 0;
1324 ch->rx_ring.tail = 0;
1325 ch->rx_ring.count = 0;
1326 ch->rx_ring.inuse_cnt = 0;
1327
1328 return ch;
1329 }
1330
1331 /*
1332 * riocm_ch_create - creates a new channel object and allocates ID for it
1333 * @ch_num: channel ID (1 ... RIOCM_MAX_CHNUM, 0 = automatic)
1334 *
1335 * Allocates and initializes a new channel object. If the parameter ch_num > 0
1336 * and is within the valid range, riocm_ch_create tries to allocate the
1337 * specified ID for the new channel. If ch_num = 0, channel ID will be assigned
1338 * automatically from the range (chstart ... RIOCM_MAX_CHNUM).
1339 * Module parameter 'chstart' defines start of an ID range available for dynamic
1340 * allocation. Range below 'chstart' is reserved for pre-defined ID numbers.
1341 * Available channel numbers are limited by 16-bit size of channel numbers used
1342 * in the packet header.
1343 *
1344 * Return value: PTR to rio_channel structure if successful (with channel number
1345 * updated via pointer) or error-valued pointer if error.
1346 */
riocm_ch_create(u16 * ch_num)1347 static struct rio_channel *riocm_ch_create(u16 *ch_num)
1348 {
1349 struct rio_channel *ch = NULL;
1350
1351 ch = riocm_ch_alloc(*ch_num);
1352
1353 if (IS_ERR(ch))
1354 riocm_debug(CHOP, "Failed to allocate channel %d (err=%ld)",
1355 *ch_num, PTR_ERR(ch));
1356 else
1357 *ch_num = ch->id;
1358
1359 return ch;
1360 }
1361
1362 /*
1363 * riocm_ch_free - channel object release routine
1364 * @ref: pointer to a channel's kref structure
1365 */
riocm_ch_free(struct kref * ref)1366 static void riocm_ch_free(struct kref *ref)
1367 {
1368 struct rio_channel *ch = container_of(ref, struct rio_channel, ref);
1369 int i;
1370
1371 riocm_debug(CHOP, "(ch_%d)", ch->id);
1372
1373 if (ch->rx_ring.inuse_cnt) {
1374 for (i = 0;
1375 i < RIOCM_RX_RING_SIZE && ch->rx_ring.inuse_cnt; i++) {
1376 if (ch->rx_ring.inuse[i] != NULL) {
1377 kfree(ch->rx_ring.inuse[i]);
1378 ch->rx_ring.inuse_cnt--;
1379 }
1380 }
1381 }
1382
1383 if (ch->rx_ring.count)
1384 for (i = 0; i < RIOCM_RX_RING_SIZE && ch->rx_ring.count; i++) {
1385 if (ch->rx_ring.buf[i] != NULL) {
1386 kfree(ch->rx_ring.buf[i]);
1387 ch->rx_ring.count--;
1388 }
1389 }
1390
1391 complete(&ch->comp_close);
1392 }
1393
riocm_send_close(struct rio_channel * ch)1394 static int riocm_send_close(struct rio_channel *ch)
1395 {
1396 struct rio_ch_chan_hdr *hdr;
1397 int ret;
1398
1399 /*
1400 * Send CH_CLOSE notification to the remote RapidIO device
1401 */
1402
1403 hdr = kzalloc(sizeof(*hdr), GFP_KERNEL);
1404 if (hdr == NULL)
1405 return -ENOMEM;
1406
1407 hdr->bhdr.src_id = htonl(ch->loc_destid);
1408 hdr->bhdr.dst_id = htonl(ch->rem_destid);
1409 hdr->bhdr.src_mbox = cmbox;
1410 hdr->bhdr.dst_mbox = cmbox;
1411 hdr->bhdr.type = RIO_CM_CHAN;
1412 hdr->ch_op = CM_CONN_CLOSE;
1413 hdr->dst_ch = htons(ch->rem_channel);
1414 hdr->src_ch = htons(ch->id);
1415
1416 /* ATTN: the function call below relies on the fact that underlying
1417 * add_outb_message() routine copies TX data into its internal transfer
1418 * buffer. Needs to be reviewed if switched to direct buffer mode.
1419 */
1420 ret = riocm_post_send(ch->cmdev, ch->rdev, hdr, sizeof(*hdr));
1421
1422 if (ret == -EBUSY && !riocm_queue_req(ch->cmdev, ch->rdev,
1423 hdr, sizeof(*hdr)))
1424 return 0;
1425 kfree(hdr);
1426
1427 if (ret)
1428 riocm_error("ch(%d) send CLOSE failed (ret=%d)", ch->id, ret);
1429
1430 return ret;
1431 }
1432
1433 /*
1434 * riocm_ch_close - closes a channel object with specified ID (by local request)
1435 * @ch: channel to be closed
1436 */
riocm_ch_close(struct rio_channel * ch)1437 static int riocm_ch_close(struct rio_channel *ch)
1438 {
1439 unsigned long tmo = msecs_to_jiffies(3000);
1440 enum rio_cm_state state;
1441 long wret;
1442 int ret = 0;
1443
1444 riocm_debug(CHOP, "ch_%d by %s(%d)",
1445 ch->id, current->comm, task_pid_nr(current));
1446
1447 state = riocm_exch(ch, RIO_CM_DESTROYING);
1448 if (state == RIO_CM_CONNECTED)
1449 riocm_send_close(ch);
1450
1451 complete_all(&ch->comp);
1452
1453 riocm_put_channel(ch);
1454 wret = wait_for_completion_interruptible_timeout(&ch->comp_close, tmo);
1455
1456 riocm_debug(WAIT, "wait on %d returns %ld", ch->id, wret);
1457
1458 if (wret == 0) {
1459 /* Timeout on wait occurred */
1460 riocm_debug(CHOP, "%s(%d) timed out waiting for ch %d",
1461 current->comm, task_pid_nr(current), ch->id);
1462 ret = -ETIMEDOUT;
1463 } else if (wret == -ERESTARTSYS) {
1464 /* Wait_for_completion was interrupted by a signal */
1465 riocm_debug(CHOP, "%s(%d) wait for ch %d was interrupted",
1466 current->comm, task_pid_nr(current), ch->id);
1467 ret = -EINTR;
1468 }
1469
1470 if (!ret) {
1471 riocm_debug(CHOP, "ch_%d resources released", ch->id);
1472 kfree(ch);
1473 } else {
1474 riocm_debug(CHOP, "failed to release ch_%d resources", ch->id);
1475 }
1476
1477 return ret;
1478 }
1479
1480 /*
1481 * riocm_cdev_open() - Open character device
1482 */
riocm_cdev_open(struct inode * inode,struct file * filp)1483 static int riocm_cdev_open(struct inode *inode, struct file *filp)
1484 {
1485 riocm_debug(INIT, "by %s(%d) filp=%p ",
1486 current->comm, task_pid_nr(current), filp);
1487
1488 if (list_empty(&cm_dev_list))
1489 return -ENODEV;
1490
1491 return 0;
1492 }
1493
1494 /*
1495 * riocm_cdev_release() - Release character device
1496 */
riocm_cdev_release(struct inode * inode,struct file * filp)1497 static int riocm_cdev_release(struct inode *inode, struct file *filp)
1498 {
1499 struct rio_channel *ch, *_c;
1500 unsigned int i;
1501 LIST_HEAD(list);
1502
1503 riocm_debug(EXIT, "by %s(%d) filp=%p",
1504 current->comm, task_pid_nr(current), filp);
1505
1506 /* Check if there are channels associated with this file descriptor */
1507 spin_lock_bh(&idr_lock);
1508 idr_for_each_entry(&ch_idr, ch, i) {
1509 if (ch && ch->filp == filp) {
1510 riocm_debug(EXIT, "ch_%d not released by %s(%d)",
1511 ch->id, current->comm,
1512 task_pid_nr(current));
1513 idr_remove(&ch_idr, ch->id);
1514 list_add(&ch->ch_node, &list);
1515 }
1516 }
1517 spin_unlock_bh(&idr_lock);
1518
1519 if (!list_empty(&list)) {
1520 list_for_each_entry_safe(ch, _c, &list, ch_node) {
1521 list_del(&ch->ch_node);
1522 riocm_ch_close(ch);
1523 }
1524 }
1525
1526 return 0;
1527 }
1528
1529 /*
1530 * cm_ep_get_list_size() - Reports number of endpoints in the network
1531 */
cm_ep_get_list_size(void __user * arg)1532 static int cm_ep_get_list_size(void __user *arg)
1533 {
1534 u32 __user *p = arg;
1535 u32 mport_id;
1536 u32 count = 0;
1537 struct cm_dev *cm;
1538
1539 if (get_user(mport_id, p))
1540 return -EFAULT;
1541 if (mport_id >= RIO_MAX_MPORTS)
1542 return -EINVAL;
1543
1544 /* Find a matching cm_dev object */
1545 down_read(&rdev_sem);
1546 list_for_each_entry(cm, &cm_dev_list, list) {
1547 if (cm->mport->id == mport_id) {
1548 count = cm->npeers;
1549 up_read(&rdev_sem);
1550 if (copy_to_user(arg, &count, sizeof(u32)))
1551 return -EFAULT;
1552 return 0;
1553 }
1554 }
1555 up_read(&rdev_sem);
1556
1557 return -ENODEV;
1558 }
1559
1560 /*
1561 * cm_ep_get_list() - Returns list of attached endpoints
1562 */
cm_ep_get_list(void __user * arg)1563 static int cm_ep_get_list(void __user *arg)
1564 {
1565 struct cm_dev *cm;
1566 struct cm_peer *peer;
1567 u32 info[2];
1568 void *buf;
1569 u32 nent;
1570 u32 *entry_ptr;
1571 u32 i = 0;
1572 int ret = 0;
1573
1574 if (copy_from_user(&info, arg, sizeof(info)))
1575 return -EFAULT;
1576
1577 if (info[1] >= RIO_MAX_MPORTS || info[0] > RIOCM_MAX_EP_COUNT)
1578 return -EINVAL;
1579
1580 /* Find a matching cm_dev object */
1581 down_read(&rdev_sem);
1582 list_for_each_entry(cm, &cm_dev_list, list)
1583 if (cm->mport->id == (u8)info[1])
1584 goto found;
1585
1586 up_read(&rdev_sem);
1587 return -ENODEV;
1588
1589 found:
1590 nent = min(info[0], cm->npeers);
1591 buf = kcalloc(nent + 2, sizeof(u32), GFP_KERNEL);
1592 if (!buf) {
1593 up_read(&rdev_sem);
1594 return -ENOMEM;
1595 }
1596
1597 entry_ptr = (u32 *)((uintptr_t)buf + 2*sizeof(u32));
1598
1599 list_for_each_entry(peer, &cm->peers, node) {
1600 *entry_ptr = (u32)peer->rdev->destid;
1601 entry_ptr++;
1602 if (++i == nent)
1603 break;
1604 }
1605 up_read(&rdev_sem);
1606
1607 ((u32 *)buf)[0] = i; /* report an updated number of entries */
1608 ((u32 *)buf)[1] = info[1]; /* put back an mport ID */
1609 if (copy_to_user(arg, buf, sizeof(u32) * (info[0] + 2)))
1610 ret = -EFAULT;
1611
1612 kfree(buf);
1613 return ret;
1614 }
1615
1616 /*
1617 * cm_mport_get_list() - Returns list of available local mport devices
1618 */
cm_mport_get_list(void __user * arg)1619 static int cm_mport_get_list(void __user *arg)
1620 {
1621 int ret = 0;
1622 u32 entries;
1623 void *buf;
1624 struct cm_dev *cm;
1625 u32 *entry_ptr;
1626 int count = 0;
1627
1628 if (copy_from_user(&entries, arg, sizeof(entries)))
1629 return -EFAULT;
1630 if (entries == 0 || entries > RIO_MAX_MPORTS)
1631 return -EINVAL;
1632 buf = kcalloc(entries + 1, sizeof(u32), GFP_KERNEL);
1633 if (!buf)
1634 return -ENOMEM;
1635
1636 /* Scan all registered cm_dev objects */
1637 entry_ptr = (u32 *)((uintptr_t)buf + sizeof(u32));
1638 down_read(&rdev_sem);
1639 list_for_each_entry(cm, &cm_dev_list, list) {
1640 if (count++ < entries) {
1641 *entry_ptr = (cm->mport->id << 16) |
1642 cm->mport->host_deviceid;
1643 entry_ptr++;
1644 }
1645 }
1646 up_read(&rdev_sem);
1647
1648 *((u32 *)buf) = count; /* report a real number of entries */
1649 if (copy_to_user(arg, buf, sizeof(u32) * (count + 1)))
1650 ret = -EFAULT;
1651
1652 kfree(buf);
1653 return ret;
1654 }
1655
1656 /*
1657 * cm_chan_create() - Create a message exchange channel
1658 */
cm_chan_create(struct file * filp,void __user * arg)1659 static int cm_chan_create(struct file *filp, void __user *arg)
1660 {
1661 u16 __user *p = arg;
1662 u16 ch_num;
1663 struct rio_channel *ch;
1664
1665 if (get_user(ch_num, p))
1666 return -EFAULT;
1667
1668 riocm_debug(CHOP, "ch_%d requested by %s(%d)",
1669 ch_num, current->comm, task_pid_nr(current));
1670 ch = riocm_ch_create(&ch_num);
1671 if (IS_ERR(ch))
1672 return PTR_ERR(ch);
1673
1674 ch->filp = filp;
1675 riocm_debug(CHOP, "ch_%d created by %s(%d)",
1676 ch_num, current->comm, task_pid_nr(current));
1677 return put_user(ch_num, p);
1678 }
1679
1680 /*
1681 * cm_chan_close() - Close channel
1682 * @filp: Pointer to file object
1683 * @arg: Channel to close
1684 */
cm_chan_close(struct file * filp,void __user * arg)1685 static int cm_chan_close(struct file *filp, void __user *arg)
1686 {
1687 u16 __user *p = arg;
1688 u16 ch_num;
1689 struct rio_channel *ch;
1690
1691 if (get_user(ch_num, p))
1692 return -EFAULT;
1693
1694 riocm_debug(CHOP, "ch_%d by %s(%d)",
1695 ch_num, current->comm, task_pid_nr(current));
1696
1697 spin_lock_bh(&idr_lock);
1698 ch = idr_find(&ch_idr, ch_num);
1699 if (!ch) {
1700 spin_unlock_bh(&idr_lock);
1701 return 0;
1702 }
1703 if (ch->filp != filp) {
1704 spin_unlock_bh(&idr_lock);
1705 return -EINVAL;
1706 }
1707 idr_remove(&ch_idr, ch->id);
1708 spin_unlock_bh(&idr_lock);
1709
1710 return riocm_ch_close(ch);
1711 }
1712
1713 /*
1714 * cm_chan_bind() - Bind channel
1715 * @arg: Channel number
1716 */
cm_chan_bind(void __user * arg)1717 static int cm_chan_bind(void __user *arg)
1718 {
1719 struct rio_cm_channel chan;
1720
1721 if (copy_from_user(&chan, arg, sizeof(chan)))
1722 return -EFAULT;
1723 if (chan.mport_id >= RIO_MAX_MPORTS)
1724 return -EINVAL;
1725
1726 return riocm_ch_bind(chan.id, chan.mport_id, NULL);
1727 }
1728
1729 /*
1730 * cm_chan_listen() - Listen on channel
1731 * @arg: Channel number
1732 */
cm_chan_listen(void __user * arg)1733 static int cm_chan_listen(void __user *arg)
1734 {
1735 u16 __user *p = arg;
1736 u16 ch_num;
1737
1738 if (get_user(ch_num, p))
1739 return -EFAULT;
1740
1741 return riocm_ch_listen(ch_num);
1742 }
1743
1744 /*
1745 * cm_chan_accept() - Accept incoming connection
1746 * @filp: Pointer to file object
1747 * @arg: Channel number
1748 */
cm_chan_accept(struct file * filp,void __user * arg)1749 static int cm_chan_accept(struct file *filp, void __user *arg)
1750 {
1751 struct rio_cm_accept param;
1752 long accept_to;
1753 struct rio_channel *ch;
1754
1755 if (copy_from_user(¶m, arg, sizeof(param)))
1756 return -EFAULT;
1757
1758 riocm_debug(CHOP, "on ch_%d by %s(%d)",
1759 param.ch_num, current->comm, task_pid_nr(current));
1760
1761 accept_to = param.wait_to ?
1762 msecs_to_jiffies(param.wait_to) : 0;
1763
1764 ch = riocm_ch_accept(param.ch_num, ¶m.ch_num, accept_to);
1765 if (IS_ERR(ch))
1766 return PTR_ERR(ch);
1767 ch->filp = filp;
1768
1769 riocm_debug(CHOP, "new ch_%d for %s(%d)",
1770 ch->id, current->comm, task_pid_nr(current));
1771
1772 if (copy_to_user(arg, ¶m, sizeof(param)))
1773 return -EFAULT;
1774 return 0;
1775 }
1776
1777 /*
1778 * cm_chan_connect() - Connect on channel
1779 * @arg: Channel information
1780 */
cm_chan_connect(void __user * arg)1781 static int cm_chan_connect(void __user *arg)
1782 {
1783 struct rio_cm_channel chan;
1784 struct cm_dev *cm;
1785 struct cm_peer *peer;
1786 int ret = -ENODEV;
1787
1788 if (copy_from_user(&chan, arg, sizeof(chan)))
1789 return -EFAULT;
1790 if (chan.mport_id >= RIO_MAX_MPORTS)
1791 return -EINVAL;
1792
1793 down_read(&rdev_sem);
1794
1795 /* Find matching cm_dev object */
1796 list_for_each_entry(cm, &cm_dev_list, list) {
1797 if (cm->mport->id == chan.mport_id) {
1798 ret = 0;
1799 break;
1800 }
1801 }
1802
1803 if (ret)
1804 goto err_out;
1805
1806 if (chan.remote_destid >= RIO_ANY_DESTID(cm->mport->sys_size)) {
1807 ret = -EINVAL;
1808 goto err_out;
1809 }
1810
1811 /* Find corresponding RapidIO endpoint device object */
1812 ret = -ENODEV;
1813
1814 list_for_each_entry(peer, &cm->peers, node) {
1815 if (peer->rdev->destid == chan.remote_destid) {
1816 ret = 0;
1817 break;
1818 }
1819 }
1820
1821 if (ret)
1822 goto err_out;
1823
1824 up_read(&rdev_sem);
1825
1826 return riocm_ch_connect(chan.id, cm, peer, chan.remote_channel);
1827 err_out:
1828 up_read(&rdev_sem);
1829 return ret;
1830 }
1831
1832 /*
1833 * cm_chan_msg_send() - Send a message through channel
1834 * @arg: Outbound message information
1835 */
cm_chan_msg_send(void __user * arg)1836 static int cm_chan_msg_send(void __user *arg)
1837 {
1838 struct rio_cm_msg msg;
1839 void *buf;
1840 int ret;
1841
1842 if (copy_from_user(&msg, arg, sizeof(msg)))
1843 return -EFAULT;
1844 if (msg.size > RIO_MAX_MSG_SIZE)
1845 return -EINVAL;
1846
1847 buf = memdup_user((void __user *)(uintptr_t)msg.msg, msg.size);
1848 if (IS_ERR(buf))
1849 return PTR_ERR(buf);
1850
1851 ret = riocm_ch_send(msg.ch_num, buf, msg.size);
1852
1853 kfree(buf);
1854 return ret;
1855 }
1856
1857 /*
1858 * cm_chan_msg_rcv() - Receive a message through channel
1859 * @arg: Inbound message information
1860 */
cm_chan_msg_rcv(void __user * arg)1861 static int cm_chan_msg_rcv(void __user *arg)
1862 {
1863 struct rio_cm_msg msg;
1864 struct rio_channel *ch;
1865 void *buf;
1866 long rxto;
1867 int ret = 0, msg_size;
1868
1869 if (copy_from_user(&msg, arg, sizeof(msg)))
1870 return -EFAULT;
1871
1872 if (msg.ch_num == 0 || msg.size == 0)
1873 return -EINVAL;
1874
1875 ch = riocm_get_channel(msg.ch_num);
1876 if (!ch)
1877 return -ENODEV;
1878
1879 rxto = msg.rxto ? msecs_to_jiffies(msg.rxto) : MAX_SCHEDULE_TIMEOUT;
1880
1881 ret = riocm_ch_receive(ch, &buf, rxto);
1882 if (ret)
1883 goto out;
1884
1885 msg_size = min(msg.size, (u16)(RIO_MAX_MSG_SIZE));
1886
1887 if (copy_to_user((void __user *)(uintptr_t)msg.msg, buf, msg_size))
1888 ret = -EFAULT;
1889
1890 riocm_ch_free_rxbuf(ch, buf);
1891 out:
1892 riocm_put_channel(ch);
1893 return ret;
1894 }
1895
1896 /*
1897 * riocm_cdev_ioctl() - IOCTL requests handler
1898 */
1899 static long
riocm_cdev_ioctl(struct file * filp,unsigned int cmd,unsigned long arg)1900 riocm_cdev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
1901 {
1902 switch (cmd) {
1903 case RIO_CM_EP_GET_LIST_SIZE:
1904 return cm_ep_get_list_size((void __user *)arg);
1905 case RIO_CM_EP_GET_LIST:
1906 return cm_ep_get_list((void __user *)arg);
1907 case RIO_CM_CHAN_CREATE:
1908 return cm_chan_create(filp, (void __user *)arg);
1909 case RIO_CM_CHAN_CLOSE:
1910 return cm_chan_close(filp, (void __user *)arg);
1911 case RIO_CM_CHAN_BIND:
1912 return cm_chan_bind((void __user *)arg);
1913 case RIO_CM_CHAN_LISTEN:
1914 return cm_chan_listen((void __user *)arg);
1915 case RIO_CM_CHAN_ACCEPT:
1916 return cm_chan_accept(filp, (void __user *)arg);
1917 case RIO_CM_CHAN_CONNECT:
1918 return cm_chan_connect((void __user *)arg);
1919 case RIO_CM_CHAN_SEND:
1920 return cm_chan_msg_send((void __user *)arg);
1921 case RIO_CM_CHAN_RECEIVE:
1922 return cm_chan_msg_rcv((void __user *)arg);
1923 case RIO_CM_MPORT_GET_LIST:
1924 return cm_mport_get_list((void __user *)arg);
1925 default:
1926 break;
1927 }
1928
1929 return -EINVAL;
1930 }
1931
1932 static const struct file_operations riocm_cdev_fops = {
1933 .owner = THIS_MODULE,
1934 .open = riocm_cdev_open,
1935 .release = riocm_cdev_release,
1936 .unlocked_ioctl = riocm_cdev_ioctl,
1937 };
1938
1939 /*
1940 * riocm_add_dev - add new remote RapidIO device into channel management core
1941 * @dev: device object associated with RapidIO device
1942 * @sif: subsystem interface
1943 *
1944 * Adds the specified RapidIO device (if applicable) into peers list of
1945 * the corresponding channel management device (cm_dev).
1946 */
riocm_add_dev(struct device * dev,struct subsys_interface * sif)1947 static int riocm_add_dev(struct device *dev, struct subsys_interface *sif)
1948 {
1949 struct cm_peer *peer;
1950 struct rio_dev *rdev = to_rio_dev(dev);
1951 struct cm_dev *cm;
1952
1953 /* Check if the remote device has capabilities required to support CM */
1954 if (!dev_cm_capable(rdev))
1955 return 0;
1956
1957 riocm_debug(RDEV, "(%s)", rio_name(rdev));
1958
1959 peer = kmalloc(sizeof(*peer), GFP_KERNEL);
1960 if (!peer)
1961 return -ENOMEM;
1962
1963 /* Find a corresponding cm_dev object */
1964 down_write(&rdev_sem);
1965 list_for_each_entry(cm, &cm_dev_list, list) {
1966 if (cm->mport == rdev->net->hport)
1967 goto found;
1968 }
1969
1970 up_write(&rdev_sem);
1971 kfree(peer);
1972 return -ENODEV;
1973
1974 found:
1975 peer->rdev = rdev;
1976 list_add_tail(&peer->node, &cm->peers);
1977 cm->npeers++;
1978
1979 up_write(&rdev_sem);
1980 return 0;
1981 }
1982
1983 /*
1984 * riocm_remove_dev - remove remote RapidIO device from channel management core
1985 * @dev: device object associated with RapidIO device
1986 * @sif: subsystem interface
1987 *
1988 * Removes the specified RapidIO device (if applicable) from peers list of
1989 * the corresponding channel management device (cm_dev).
1990 */
riocm_remove_dev(struct device * dev,struct subsys_interface * sif)1991 static void riocm_remove_dev(struct device *dev, struct subsys_interface *sif)
1992 {
1993 struct rio_dev *rdev = to_rio_dev(dev);
1994 struct cm_dev *cm;
1995 struct cm_peer *peer;
1996 struct rio_channel *ch, *_c;
1997 unsigned int i;
1998 bool found = false;
1999 LIST_HEAD(list);
2000
2001 /* Check if the remote device has capabilities required to support CM */
2002 if (!dev_cm_capable(rdev))
2003 return;
2004
2005 riocm_debug(RDEV, "(%s)", rio_name(rdev));
2006
2007 /* Find matching cm_dev object */
2008 down_write(&rdev_sem);
2009 list_for_each_entry(cm, &cm_dev_list, list) {
2010 if (cm->mport == rdev->net->hport) {
2011 found = true;
2012 break;
2013 }
2014 }
2015
2016 if (!found) {
2017 up_write(&rdev_sem);
2018 return;
2019 }
2020
2021 /* Remove remote device from the list of peers */
2022 found = false;
2023 list_for_each_entry(peer, &cm->peers, node) {
2024 if (peer->rdev == rdev) {
2025 riocm_debug(RDEV, "removing peer %s", rio_name(rdev));
2026 found = true;
2027 list_del(&peer->node);
2028 cm->npeers--;
2029 kfree(peer);
2030 break;
2031 }
2032 }
2033
2034 up_write(&rdev_sem);
2035
2036 if (!found)
2037 return;
2038
2039 /*
2040 * Release channels associated with this peer
2041 */
2042
2043 spin_lock_bh(&idr_lock);
2044 idr_for_each_entry(&ch_idr, ch, i) {
2045 if (ch && ch->rdev == rdev) {
2046 if (atomic_read(&rdev->state) != RIO_DEVICE_SHUTDOWN)
2047 riocm_exch(ch, RIO_CM_DISCONNECT);
2048 idr_remove(&ch_idr, ch->id);
2049 list_add(&ch->ch_node, &list);
2050 }
2051 }
2052 spin_unlock_bh(&idr_lock);
2053
2054 if (!list_empty(&list)) {
2055 list_for_each_entry_safe(ch, _c, &list, ch_node) {
2056 list_del(&ch->ch_node);
2057 riocm_ch_close(ch);
2058 }
2059 }
2060 }
2061
2062 /*
2063 * riocm_cdev_add() - Create rio_cm char device
2064 * @devno: device number assigned to device (MAJ + MIN)
2065 */
riocm_cdev_add(dev_t devno)2066 static int riocm_cdev_add(dev_t devno)
2067 {
2068 int ret;
2069
2070 cdev_init(&riocm_cdev.cdev, &riocm_cdev_fops);
2071 riocm_cdev.cdev.owner = THIS_MODULE;
2072 ret = cdev_add(&riocm_cdev.cdev, devno, 1);
2073 if (ret < 0) {
2074 riocm_error("Cannot register a device with error %d", ret);
2075 return ret;
2076 }
2077
2078 riocm_cdev.dev = device_create(dev_class, NULL, devno, NULL, DEV_NAME);
2079 if (IS_ERR(riocm_cdev.dev)) {
2080 cdev_del(&riocm_cdev.cdev);
2081 return PTR_ERR(riocm_cdev.dev);
2082 }
2083
2084 riocm_debug(MPORT, "Added %s cdev(%d:%d)",
2085 DEV_NAME, MAJOR(devno), MINOR(devno));
2086
2087 return 0;
2088 }
2089
2090 /*
2091 * riocm_add_mport - add new local mport device into channel management core
2092 * @dev: device object associated with mport
2093 *
2094 * When a new mport device is added, CM immediately reserves inbound and
2095 * outbound RapidIO mailboxes that will be used.
2096 */
riocm_add_mport(struct device * dev)2097 static int riocm_add_mport(struct device *dev)
2098 {
2099 int rc;
2100 int i;
2101 struct cm_dev *cm;
2102 struct rio_mport *mport = to_rio_mport(dev);
2103
2104 riocm_debug(MPORT, "add mport %s", mport->name);
2105
2106 cm = kzalloc(sizeof(*cm), GFP_KERNEL);
2107 if (!cm)
2108 return -ENOMEM;
2109
2110 cm->mport = mport;
2111
2112 rc = rio_request_outb_mbox(mport, cm, cmbox,
2113 RIOCM_TX_RING_SIZE, riocm_outb_msg_event);
2114 if (rc) {
2115 riocm_error("failed to allocate OBMBOX_%d on %s",
2116 cmbox, mport->name);
2117 kfree(cm);
2118 return -ENODEV;
2119 }
2120
2121 rc = rio_request_inb_mbox(mport, cm, cmbox,
2122 RIOCM_RX_RING_SIZE, riocm_inb_msg_event);
2123 if (rc) {
2124 riocm_error("failed to allocate IBMBOX_%d on %s",
2125 cmbox, mport->name);
2126 rio_release_outb_mbox(mport, cmbox);
2127 kfree(cm);
2128 return -ENODEV;
2129 }
2130
2131 cm->rx_wq = create_workqueue(DRV_NAME "/rxq");
2132 if (!cm->rx_wq) {
2133 rio_release_inb_mbox(mport, cmbox);
2134 rio_release_outb_mbox(mport, cmbox);
2135 kfree(cm);
2136 return -ENOMEM;
2137 }
2138
2139 /*
2140 * Allocate and register inbound messaging buffers to be ready
2141 * to receive channel and system management requests
2142 */
2143 for (i = 0; i < RIOCM_RX_RING_SIZE; i++)
2144 cm->rx_buf[i] = NULL;
2145
2146 cm->rx_slots = RIOCM_RX_RING_SIZE;
2147 mutex_init(&cm->rx_lock);
2148 riocm_rx_fill(cm, RIOCM_RX_RING_SIZE);
2149 INIT_WORK(&cm->rx_work, rio_ibmsg_handler);
2150
2151 cm->tx_slot = 0;
2152 cm->tx_cnt = 0;
2153 cm->tx_ack_slot = 0;
2154 spin_lock_init(&cm->tx_lock);
2155
2156 INIT_LIST_HEAD(&cm->peers);
2157 cm->npeers = 0;
2158 INIT_LIST_HEAD(&cm->tx_reqs);
2159
2160 down_write(&rdev_sem);
2161 list_add_tail(&cm->list, &cm_dev_list);
2162 up_write(&rdev_sem);
2163
2164 return 0;
2165 }
2166
2167 /*
2168 * riocm_remove_mport - remove local mport device from channel management core
2169 * @dev: device object associated with mport
2170 *
2171 * Removes a local mport device from the list of registered devices that provide
2172 * channel management services. Returns an error if the specified mport is not
2173 * registered with the CM core.
2174 */
riocm_remove_mport(struct device * dev)2175 static void riocm_remove_mport(struct device *dev)
2176 {
2177 struct rio_mport *mport = to_rio_mport(dev);
2178 struct cm_dev *cm;
2179 struct cm_peer *peer, *temp;
2180 struct rio_channel *ch, *_c;
2181 unsigned int i;
2182 bool found = false;
2183 LIST_HEAD(list);
2184
2185 riocm_debug(MPORT, "%s", mport->name);
2186
2187 /* Find a matching cm_dev object */
2188 down_write(&rdev_sem);
2189 list_for_each_entry(cm, &cm_dev_list, list) {
2190 if (cm->mport == mport) {
2191 list_del(&cm->list);
2192 found = true;
2193 break;
2194 }
2195 }
2196 up_write(&rdev_sem);
2197 if (!found)
2198 return;
2199
2200 flush_workqueue(cm->rx_wq);
2201 destroy_workqueue(cm->rx_wq);
2202
2203 /* Release channels bound to this mport */
2204 spin_lock_bh(&idr_lock);
2205 idr_for_each_entry(&ch_idr, ch, i) {
2206 if (ch->cmdev == cm) {
2207 riocm_debug(RDEV, "%s drop ch_%d",
2208 mport->name, ch->id);
2209 idr_remove(&ch_idr, ch->id);
2210 list_add(&ch->ch_node, &list);
2211 }
2212 }
2213 spin_unlock_bh(&idr_lock);
2214
2215 if (!list_empty(&list)) {
2216 list_for_each_entry_safe(ch, _c, &list, ch_node) {
2217 list_del(&ch->ch_node);
2218 riocm_ch_close(ch);
2219 }
2220 }
2221
2222 rio_release_inb_mbox(mport, cmbox);
2223 rio_release_outb_mbox(mport, cmbox);
2224
2225 /* Remove and free peer entries */
2226 if (!list_empty(&cm->peers))
2227 riocm_debug(RDEV, "ATTN: peer list not empty");
2228 list_for_each_entry_safe(peer, temp, &cm->peers, node) {
2229 riocm_debug(RDEV, "removing peer %s", rio_name(peer->rdev));
2230 list_del(&peer->node);
2231 kfree(peer);
2232 }
2233
2234 riocm_rx_free(cm);
2235 kfree(cm);
2236 riocm_debug(MPORT, "%s done", mport->name);
2237 }
2238
rio_cm_shutdown(struct notifier_block * nb,unsigned long code,void * unused)2239 static int rio_cm_shutdown(struct notifier_block *nb, unsigned long code,
2240 void *unused)
2241 {
2242 struct rio_channel *ch;
2243 unsigned int i;
2244 LIST_HEAD(list);
2245
2246 riocm_debug(EXIT, ".");
2247
2248 /*
2249 * If there are any channels left in connected state send
2250 * close notification to the connection partner.
2251 * First build a list of channels that require a closing
2252 * notification because function riocm_send_close() should
2253 * be called outside of spinlock protected code.
2254 */
2255 spin_lock_bh(&idr_lock);
2256 idr_for_each_entry(&ch_idr, ch, i) {
2257 if (ch->state == RIO_CM_CONNECTED) {
2258 riocm_debug(EXIT, "close ch %d", ch->id);
2259 idr_remove(&ch_idr, ch->id);
2260 list_add(&ch->ch_node, &list);
2261 }
2262 }
2263 spin_unlock_bh(&idr_lock);
2264
2265 list_for_each_entry(ch, &list, ch_node)
2266 riocm_send_close(ch);
2267
2268 return NOTIFY_DONE;
2269 }
2270
2271 /*
2272 * riocm_interface handles addition/removal of remote RapidIO devices
2273 */
2274 static struct subsys_interface riocm_interface = {
2275 .name = "rio_cm",
2276 .subsys = &rio_bus_type,
2277 .add_dev = riocm_add_dev,
2278 .remove_dev = riocm_remove_dev,
2279 };
2280
2281 /*
2282 * rio_mport_interface handles addition/removal local mport devices
2283 */
2284 static struct class_interface rio_mport_interface __refdata = {
2285 .class = &rio_mport_class,
2286 .add_dev = riocm_add_mport,
2287 .remove_dev = riocm_remove_mport,
2288 };
2289
2290 static struct notifier_block rio_cm_notifier = {
2291 .notifier_call = rio_cm_shutdown,
2292 };
2293
riocm_init(void)2294 static int __init riocm_init(void)
2295 {
2296 int ret;
2297
2298 /* Create device class needed by udev */
2299 dev_class = class_create(DRV_NAME);
2300 if (IS_ERR(dev_class)) {
2301 riocm_error("Cannot create " DRV_NAME " class");
2302 return PTR_ERR(dev_class);
2303 }
2304
2305 ret = alloc_chrdev_region(&dev_number, 0, 1, DRV_NAME);
2306 if (ret) {
2307 class_destroy(dev_class);
2308 return ret;
2309 }
2310
2311 dev_major = MAJOR(dev_number);
2312 dev_minor_base = MINOR(dev_number);
2313 riocm_debug(INIT, "Registered class with %d major", dev_major);
2314
2315 /*
2316 * Register as rapidio_port class interface to get notifications about
2317 * mport additions and removals.
2318 */
2319 ret = class_interface_register(&rio_mport_interface);
2320 if (ret) {
2321 riocm_error("class_interface_register error: %d", ret);
2322 goto err_reg;
2323 }
2324
2325 /*
2326 * Register as RapidIO bus interface to get notifications about
2327 * addition/removal of remote RapidIO devices.
2328 */
2329 ret = subsys_interface_register(&riocm_interface);
2330 if (ret) {
2331 riocm_error("subsys_interface_register error: %d", ret);
2332 goto err_cl;
2333 }
2334
2335 ret = register_reboot_notifier(&rio_cm_notifier);
2336 if (ret) {
2337 riocm_error("failed to register reboot notifier (err=%d)", ret);
2338 goto err_sif;
2339 }
2340
2341 ret = riocm_cdev_add(dev_number);
2342 if (ret) {
2343 unregister_reboot_notifier(&rio_cm_notifier);
2344 ret = -ENODEV;
2345 goto err_sif;
2346 }
2347
2348 return 0;
2349 err_sif:
2350 subsys_interface_unregister(&riocm_interface);
2351 err_cl:
2352 class_interface_unregister(&rio_mport_interface);
2353 err_reg:
2354 unregister_chrdev_region(dev_number, 1);
2355 class_destroy(dev_class);
2356 return ret;
2357 }
2358
riocm_exit(void)2359 static void __exit riocm_exit(void)
2360 {
2361 riocm_debug(EXIT, "enter");
2362 unregister_reboot_notifier(&rio_cm_notifier);
2363 subsys_interface_unregister(&riocm_interface);
2364 class_interface_unregister(&rio_mport_interface);
2365 idr_destroy(&ch_idr);
2366
2367 device_unregister(riocm_cdev.dev);
2368 cdev_del(&(riocm_cdev.cdev));
2369
2370 class_destroy(dev_class);
2371 unregister_chrdev_region(dev_number, 1);
2372 }
2373
2374 late_initcall(riocm_init);
2375 module_exit(riocm_exit);
2376