1 /*
2  * Copyright (c) 2007, 2008, 2009 QLogic Corporation. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 #include <linux/mm.h>
33 #include <linux/types.h>
34 #include <linux/device.h>
35 #include <linux/dmapool.h>
36 #include <linux/slab.h>
37 #include <linux/list.h>
38 #include <linux/highmem.h>
39 #include <linux/io.h>
40 #include <linux/uio.h>
41 #include <linux/rbtree.h>
42 #include <linux/spinlock.h>
43 #include <linux/delay.h>
44 
45 #include "qib.h"
46 #include "qib_user_sdma.h"
47 
48 /* minimum size of header */
49 #define QIB_USER_SDMA_MIN_HEADER_LENGTH 64
50 /* expected size of headers (for dma_pool) */
51 #define QIB_USER_SDMA_EXP_HEADER_LENGTH 64
52 /* attempt to drain the queue for 5secs */
53 #define QIB_USER_SDMA_DRAIN_TIMEOUT 250
54 
55 /*
56  * track how many times a process open this driver.
57  */
58 static struct rb_root qib_user_sdma_rb_root = RB_ROOT;
59 
60 struct qib_user_sdma_rb_node {
61 	struct rb_node node;
62 	int refcount;
63 	pid_t pid;
64 };
65 
66 struct qib_user_sdma_pkt {
67 	struct list_head list;  /* list element */
68 
69 	u8  tiddma;		/* if this is NEW tid-sdma */
70 	u8  largepkt;		/* this is large pkt from kmalloc */
71 	u16 frag_size;		/* frag size used by PSM */
72 	u16 index;              /* last header index or push index */
73 	u16 naddr;              /* dimension of addr (1..3) ... */
74 	u16 addrlimit;		/* addr array size */
75 	u16 tidsmidx;		/* current tidsm index */
76 	u16 tidsmcount;		/* tidsm array item count */
77 	u16 payload_size;	/* payload size so far for header */
78 	u32 bytes_togo;		/* bytes for processing */
79 	u32 counter;            /* sdma pkts queued counter for this entry */
80 	struct qib_tid_session_member *tidsm;	/* tid session member array */
81 	struct qib_user_sdma_queue *pq;	/* which pq this pkt belongs to */
82 	u64 added;              /* global descq number of entries */
83 
84 	struct {
85 		u16 offset;                     /* offset for kvaddr, addr */
86 		u16 length;                     /* length in page */
87 		u16 first_desc;			/* first desc */
88 		u16 last_desc;			/* last desc */
89 		u16 put_page;                   /* should we put_page? */
90 		u16 dma_mapped;                 /* is page dma_mapped? */
91 		u16 dma_length;			/* for dma_unmap_page() */
92 		u16 padding;
93 		struct page *page;              /* may be NULL (coherent mem) */
94 		void *kvaddr;                   /* FIXME: only for pio hack */
95 		dma_addr_t addr;
96 	} addr[4];   /* max pages, any more and we coalesce */
97 };
98 
99 struct qib_user_sdma_queue {
100 	/*
101 	 * pkts sent to dma engine are queued on this
102 	 * list head.  the type of the elements of this
103 	 * list are struct qib_user_sdma_pkt...
104 	 */
105 	struct list_head sent;
106 
107 	/*
108 	 * Because above list will be accessed by both process and
109 	 * signal handler, we need a spinlock for it.
110 	 */
111 	spinlock_t sent_lock ____cacheline_aligned_in_smp;
112 
113 	/* headers with expected length are allocated from here... */
114 	char header_cache_name[64];
115 	struct dma_pool *header_cache;
116 
117 	/* packets are allocated from the slab cache... */
118 	char pkt_slab_name[64];
119 	struct kmem_cache *pkt_slab;
120 
121 	/* as packets go on the queued queue, they are counted... */
122 	u32 counter;
123 	u32 sent_counter;
124 	/* pending packets, not sending yet */
125 	u32 num_pending;
126 	/* sending packets, not complete yet */
127 	u32 num_sending;
128 	/* global descq number of entry of last sending packet */
129 	u64 added;
130 
131 	/* dma page table */
132 	struct rb_root dma_pages_root;
133 
134 	struct qib_user_sdma_rb_node *sdma_rb_node;
135 
136 	/* protect everything above... */
137 	struct mutex lock;
138 };
139 
140 static struct qib_user_sdma_rb_node *
141 qib_user_sdma_rb_search(struct rb_root *root, pid_t pid)
142 {
143 	struct qib_user_sdma_rb_node *sdma_rb_node;
144 	struct rb_node *node = root->rb_node;
145 
146 	while (node) {
147 		sdma_rb_node = rb_entry(node, struct qib_user_sdma_rb_node,
148 					node);
149 		if (pid < sdma_rb_node->pid)
150 			node = node->rb_left;
151 		else if (pid > sdma_rb_node->pid)
152 			node = node->rb_right;
153 		else
154 			return sdma_rb_node;
155 	}
156 	return NULL;
157 }
158 
159 static int
160 qib_user_sdma_rb_insert(struct rb_root *root, struct qib_user_sdma_rb_node *new)
161 {
162 	struct rb_node **node = &(root->rb_node);
163 	struct rb_node *parent = NULL;
164 	struct qib_user_sdma_rb_node *got;
165 
166 	while (*node) {
167 		got = rb_entry(*node, struct qib_user_sdma_rb_node, node);
168 		parent = *node;
169 		if (new->pid < got->pid)
170 			node = &((*node)->rb_left);
171 		else if (new->pid > got->pid)
172 			node = &((*node)->rb_right);
173 		else
174 			return 0;
175 	}
176 
177 	rb_link_node(&new->node, parent, node);
178 	rb_insert_color(&new->node, root);
179 	return 1;
180 }
181 
182 struct qib_user_sdma_queue *
183 qib_user_sdma_queue_create(struct device *dev, int unit, int ctxt, int sctxt)
184 {
185 	struct qib_user_sdma_queue *pq =
186 		kmalloc(sizeof(struct qib_user_sdma_queue), GFP_KERNEL);
187 	struct qib_user_sdma_rb_node *sdma_rb_node;
188 
189 	if (!pq)
190 		goto done;
191 
192 	pq->counter = 0;
193 	pq->sent_counter = 0;
194 	pq->num_pending = 0;
195 	pq->num_sending = 0;
196 	pq->added = 0;
197 	pq->sdma_rb_node = NULL;
198 
199 	INIT_LIST_HEAD(&pq->sent);
200 	spin_lock_init(&pq->sent_lock);
201 	mutex_init(&pq->lock);
202 
203 	snprintf(pq->pkt_slab_name, sizeof(pq->pkt_slab_name),
204 		 "qib-user-sdma-pkts-%u-%02u.%02u", unit, ctxt, sctxt);
205 	pq->pkt_slab = kmem_cache_create(pq->pkt_slab_name,
206 					 sizeof(struct qib_user_sdma_pkt),
207 					 0, 0, NULL);
208 
209 	if (!pq->pkt_slab)
210 		goto err_kfree;
211 
212 	snprintf(pq->header_cache_name, sizeof(pq->header_cache_name),
213 		 "qib-user-sdma-headers-%u-%02u.%02u", unit, ctxt, sctxt);
214 	pq->header_cache = dma_pool_create(pq->header_cache_name,
215 					   dev,
216 					   QIB_USER_SDMA_EXP_HEADER_LENGTH,
217 					   4, 0);
218 	if (!pq->header_cache)
219 		goto err_slab;
220 
221 	pq->dma_pages_root = RB_ROOT;
222 
223 	sdma_rb_node = qib_user_sdma_rb_search(&qib_user_sdma_rb_root,
224 					current->pid);
225 	if (sdma_rb_node) {
226 		sdma_rb_node->refcount++;
227 	} else {
228 		sdma_rb_node = kmalloc(sizeof(
229 			struct qib_user_sdma_rb_node), GFP_KERNEL);
230 		if (!sdma_rb_node)
231 			goto err_rb;
232 
233 		sdma_rb_node->refcount = 1;
234 		sdma_rb_node->pid = current->pid;
235 
236 		qib_user_sdma_rb_insert(&qib_user_sdma_rb_root, sdma_rb_node);
237 	}
238 	pq->sdma_rb_node = sdma_rb_node;
239 
240 	goto done;
241 
242 err_rb:
243 	dma_pool_destroy(pq->header_cache);
244 err_slab:
245 	kmem_cache_destroy(pq->pkt_slab);
246 err_kfree:
247 	kfree(pq);
248 	pq = NULL;
249 
250 done:
251 	return pq;
252 }
253 
254 static void qib_user_sdma_init_frag(struct qib_user_sdma_pkt *pkt,
255 				    int i, u16 offset, u16 len,
256 				    u16 first_desc, u16 last_desc,
257 				    u16 put_page, u16 dma_mapped,
258 				    struct page *page, void *kvaddr,
259 				    dma_addr_t dma_addr, u16 dma_length)
260 {
261 	pkt->addr[i].offset = offset;
262 	pkt->addr[i].length = len;
263 	pkt->addr[i].first_desc = first_desc;
264 	pkt->addr[i].last_desc = last_desc;
265 	pkt->addr[i].put_page = put_page;
266 	pkt->addr[i].dma_mapped = dma_mapped;
267 	pkt->addr[i].page = page;
268 	pkt->addr[i].kvaddr = kvaddr;
269 	pkt->addr[i].addr = dma_addr;
270 	pkt->addr[i].dma_length = dma_length;
271 }
272 
273 static void *qib_user_sdma_alloc_header(struct qib_user_sdma_queue *pq,
274 				size_t len, dma_addr_t *dma_addr)
275 {
276 	void *hdr;
277 
278 	if (len == QIB_USER_SDMA_EXP_HEADER_LENGTH)
279 		hdr = dma_pool_alloc(pq->header_cache, GFP_KERNEL,
280 					     dma_addr);
281 	else
282 		hdr = NULL;
283 
284 	if (!hdr) {
285 		hdr = kmalloc(len, GFP_KERNEL);
286 		if (!hdr)
287 			return NULL;
288 
289 		*dma_addr = 0;
290 	}
291 
292 	return hdr;
293 }
294 
295 static int qib_user_sdma_page_to_frags(const struct qib_devdata *dd,
296 				       struct qib_user_sdma_queue *pq,
297 				       struct qib_user_sdma_pkt *pkt,
298 				       struct page *page, u16 put,
299 				       u16 offset, u16 len, void *kvaddr)
300 {
301 	__le16 *pbc16;
302 	void *pbcvaddr;
303 	struct qib_message_header *hdr;
304 	u16 newlen, pbclen, lastdesc, dma_mapped;
305 	u32 vcto;
306 	union qib_seqnum seqnum;
307 	dma_addr_t pbcdaddr;
308 	dma_addr_t dma_addr =
309 		dma_map_page(&dd->pcidev->dev,
310 			page, offset, len, DMA_TO_DEVICE);
311 	int ret = 0;
312 
313 	if (dma_mapping_error(&dd->pcidev->dev, dma_addr)) {
314 		/*
315 		 * dma mapping error, pkt has not managed
316 		 * this page yet, return the page here so
317 		 * the caller can ignore this page.
318 		 */
319 		if (put) {
320 			put_page(page);
321 		} else {
322 			/* coalesce case */
323 			kunmap(page);
324 			__free_page(page);
325 		}
326 		ret = -ENOMEM;
327 		goto done;
328 	}
329 	offset = 0;
330 	dma_mapped = 1;
331 
332 
333 next_fragment:
334 
335 	/*
336 	 * In tid-sdma, the transfer length is restricted by
337 	 * receiver side current tid page length.
338 	 */
339 	if (pkt->tiddma && len > pkt->tidsm[pkt->tidsmidx].length)
340 		newlen = pkt->tidsm[pkt->tidsmidx].length;
341 	else
342 		newlen = len;
343 
344 	/*
345 	 * Then the transfer length is restricted by MTU.
346 	 * the last descriptor flag is determined by:
347 	 * 1. the current packet is at frag size length.
348 	 * 2. the current tid page is done if tid-sdma.
349 	 * 3. there is no more byte togo if sdma.
350 	 */
351 	lastdesc = 0;
352 	if ((pkt->payload_size + newlen) >= pkt->frag_size) {
353 		newlen = pkt->frag_size - pkt->payload_size;
354 		lastdesc = 1;
355 	} else if (pkt->tiddma) {
356 		if (newlen == pkt->tidsm[pkt->tidsmidx].length)
357 			lastdesc = 1;
358 	} else {
359 		if (newlen == pkt->bytes_togo)
360 			lastdesc = 1;
361 	}
362 
363 	/* fill the next fragment in this page */
364 	qib_user_sdma_init_frag(pkt, pkt->naddr, /* index */
365 		offset, newlen,		/* offset, len */
366 		0, lastdesc,		/* first last desc */
367 		put, dma_mapped,	/* put page, dma mapped */
368 		page, kvaddr,		/* struct page, virt addr */
369 		dma_addr, len);		/* dma addr, dma length */
370 	pkt->bytes_togo -= newlen;
371 	pkt->payload_size += newlen;
372 	pkt->naddr++;
373 	if (pkt->naddr == pkt->addrlimit) {
374 		ret = -EFAULT;
375 		goto done;
376 	}
377 
378 	/* If there is no more byte togo. (lastdesc==1) */
379 	if (pkt->bytes_togo == 0) {
380 		/* The packet is done, header is not dma mapped yet.
381 		 * it should be from kmalloc */
382 		if (!pkt->addr[pkt->index].addr) {
383 			pkt->addr[pkt->index].addr =
384 				dma_map_single(&dd->pcidev->dev,
385 					pkt->addr[pkt->index].kvaddr,
386 					pkt->addr[pkt->index].dma_length,
387 					DMA_TO_DEVICE);
388 			if (dma_mapping_error(&dd->pcidev->dev,
389 					pkt->addr[pkt->index].addr)) {
390 				ret = -ENOMEM;
391 				goto done;
392 			}
393 			pkt->addr[pkt->index].dma_mapped = 1;
394 		}
395 
396 		goto done;
397 	}
398 
399 	/* If tid-sdma, advance tid info. */
400 	if (pkt->tiddma) {
401 		pkt->tidsm[pkt->tidsmidx].length -= newlen;
402 		if (pkt->tidsm[pkt->tidsmidx].length) {
403 			pkt->tidsm[pkt->tidsmidx].offset += newlen;
404 		} else {
405 			pkt->tidsmidx++;
406 			if (pkt->tidsmidx == pkt->tidsmcount) {
407 				ret = -EFAULT;
408 				goto done;
409 			}
410 		}
411 	}
412 
413 	/*
414 	 * If this is NOT the last descriptor. (newlen==len)
415 	 * the current packet is not done yet, but the current
416 	 * send side page is done.
417 	 */
418 	if (lastdesc == 0)
419 		goto done;
420 
421 	/*
422 	 * If running this driver under PSM with message size
423 	 * fitting into one transfer unit, it is not possible
424 	 * to pass this line. otherwise, it is a buggggg.
425 	 */
426 
427 	/*
428 	 * Since the current packet is done, and there are more
429 	 * bytes togo, we need to create a new sdma header, copying
430 	 * from previous sdma header and modify both.
431 	 */
432 	pbclen = pkt->addr[pkt->index].length;
433 	pbcvaddr = qib_user_sdma_alloc_header(pq, pbclen, &pbcdaddr);
434 	if (!pbcvaddr) {
435 		ret = -ENOMEM;
436 		goto done;
437 	}
438 	/* Copy the previous sdma header to new sdma header */
439 	pbc16 = (__le16 *)pkt->addr[pkt->index].kvaddr;
440 	memcpy(pbcvaddr, pbc16, pbclen);
441 
442 	/* Modify the previous sdma header */
443 	hdr = (struct qib_message_header *)&pbc16[4];
444 
445 	/* New pbc length */
446 	pbc16[0] = cpu_to_le16(le16_to_cpu(pbc16[0])-(pkt->bytes_togo>>2));
447 
448 	/* New packet length */
449 	hdr->lrh[2] = cpu_to_be16(le16_to_cpu(pbc16[0]));
450 
451 	if (pkt->tiddma) {
452 		/* turn on the header suppression */
453 		hdr->iph.pkt_flags =
454 			cpu_to_le16(le16_to_cpu(hdr->iph.pkt_flags)|0x2);
455 		/* turn off ACK_REQ: 0x04 and EXPECTED_DONE: 0x20 */
456 		hdr->flags &= ~(0x04|0x20);
457 	} else {
458 		/* turn off extra bytes: 20-21 bits */
459 		hdr->bth[0] = cpu_to_be32(be32_to_cpu(hdr->bth[0])&0xFFCFFFFF);
460 		/* turn off ACK_REQ: 0x04 */
461 		hdr->flags &= ~(0x04);
462 	}
463 
464 	/* New kdeth checksum */
465 	vcto = le32_to_cpu(hdr->iph.ver_ctxt_tid_offset);
466 	hdr->iph.chksum = cpu_to_le16(QIB_LRH_BTH +
467 		be16_to_cpu(hdr->lrh[2]) -
468 		((vcto>>16)&0xFFFF) - (vcto&0xFFFF) -
469 		le16_to_cpu(hdr->iph.pkt_flags));
470 
471 	/* The packet is done, header is not dma mapped yet.
472 	 * it should be from kmalloc */
473 	if (!pkt->addr[pkt->index].addr) {
474 		pkt->addr[pkt->index].addr =
475 			dma_map_single(&dd->pcidev->dev,
476 				pkt->addr[pkt->index].kvaddr,
477 				pkt->addr[pkt->index].dma_length,
478 				DMA_TO_DEVICE);
479 		if (dma_mapping_error(&dd->pcidev->dev,
480 				pkt->addr[pkt->index].addr)) {
481 			ret = -ENOMEM;
482 			goto done;
483 		}
484 		pkt->addr[pkt->index].dma_mapped = 1;
485 	}
486 
487 	/* Modify the new sdma header */
488 	pbc16 = (__le16 *)pbcvaddr;
489 	hdr = (struct qib_message_header *)&pbc16[4];
490 
491 	/* New pbc length */
492 	pbc16[0] = cpu_to_le16(le16_to_cpu(pbc16[0])-(pkt->payload_size>>2));
493 
494 	/* New packet length */
495 	hdr->lrh[2] = cpu_to_be16(le16_to_cpu(pbc16[0]));
496 
497 	if (pkt->tiddma) {
498 		/* Set new tid and offset for new sdma header */
499 		hdr->iph.ver_ctxt_tid_offset = cpu_to_le32(
500 			(le32_to_cpu(hdr->iph.ver_ctxt_tid_offset)&0xFF000000) +
501 			(pkt->tidsm[pkt->tidsmidx].tid<<QLOGIC_IB_I_TID_SHIFT) +
502 			(pkt->tidsm[pkt->tidsmidx].offset>>2));
503 	} else {
504 		/* Middle protocol new packet offset */
505 		hdr->uwords[2] += pkt->payload_size;
506 	}
507 
508 	/* New kdeth checksum */
509 	vcto = le32_to_cpu(hdr->iph.ver_ctxt_tid_offset);
510 	hdr->iph.chksum = cpu_to_le16(QIB_LRH_BTH +
511 		be16_to_cpu(hdr->lrh[2]) -
512 		((vcto>>16)&0xFFFF) - (vcto&0xFFFF) -
513 		le16_to_cpu(hdr->iph.pkt_flags));
514 
515 	/* Next sequence number in new sdma header */
516 	seqnum.val = be32_to_cpu(hdr->bth[2]);
517 	if (pkt->tiddma)
518 		seqnum.seq++;
519 	else
520 		seqnum.pkt++;
521 	hdr->bth[2] = cpu_to_be32(seqnum.val);
522 
523 	/* Init new sdma header. */
524 	qib_user_sdma_init_frag(pkt, pkt->naddr, /* index */
525 		0, pbclen,		/* offset, len */
526 		1, 0,			/* first last desc */
527 		0, 0,			/* put page, dma mapped */
528 		NULL, pbcvaddr,		/* struct page, virt addr */
529 		pbcdaddr, pbclen);	/* dma addr, dma length */
530 	pkt->index = pkt->naddr;
531 	pkt->payload_size = 0;
532 	pkt->naddr++;
533 	if (pkt->naddr == pkt->addrlimit) {
534 		ret = -EFAULT;
535 		goto done;
536 	}
537 
538 	/* Prepare for next fragment in this page */
539 	if (newlen != len) {
540 		if (dma_mapped) {
541 			put = 0;
542 			dma_mapped = 0;
543 			page = NULL;
544 			kvaddr = NULL;
545 		}
546 		len -= newlen;
547 		offset += newlen;
548 
549 		goto next_fragment;
550 	}
551 
552 done:
553 	return ret;
554 }
555 
556 /* we've too many pages in the iovec, coalesce to a single page */
557 static int qib_user_sdma_coalesce(const struct qib_devdata *dd,
558 				  struct qib_user_sdma_queue *pq,
559 				  struct qib_user_sdma_pkt *pkt,
560 				  const struct iovec *iov,
561 				  unsigned long niov)
562 {
563 	int ret = 0;
564 	struct page *page = alloc_page(GFP_KERNEL);
565 	void *mpage_save;
566 	char *mpage;
567 	int i;
568 	int len = 0;
569 
570 	if (!page) {
571 		ret = -ENOMEM;
572 		goto done;
573 	}
574 
575 	mpage = kmap(page);
576 	mpage_save = mpage;
577 	for (i = 0; i < niov; i++) {
578 		int cfur;
579 
580 		cfur = copy_from_user(mpage,
581 				      iov[i].iov_base, iov[i].iov_len);
582 		if (cfur) {
583 			ret = -EFAULT;
584 			goto free_unmap;
585 		}
586 
587 		mpage += iov[i].iov_len;
588 		len += iov[i].iov_len;
589 	}
590 
591 	ret = qib_user_sdma_page_to_frags(dd, pq, pkt,
592 			page, 0, 0, len, mpage_save);
593 	goto done;
594 
595 free_unmap:
596 	kunmap(page);
597 	__free_page(page);
598 done:
599 	return ret;
600 }
601 
602 /*
603  * How many pages in this iovec element?
604  */
605 static int qib_user_sdma_num_pages(const struct iovec *iov)
606 {
607 	const unsigned long addr  = (unsigned long) iov->iov_base;
608 	const unsigned long  len  = iov->iov_len;
609 	const unsigned long spage = addr & PAGE_MASK;
610 	const unsigned long epage = (addr + len - 1) & PAGE_MASK;
611 
612 	return 1 + ((epage - spage) >> PAGE_SHIFT);
613 }
614 
615 static void qib_user_sdma_free_pkt_frag(struct device *dev,
616 					struct qib_user_sdma_queue *pq,
617 					struct qib_user_sdma_pkt *pkt,
618 					int frag)
619 {
620 	const int i = frag;
621 
622 	if (pkt->addr[i].page) {
623 		/* only user data has page */
624 		if (pkt->addr[i].dma_mapped)
625 			dma_unmap_page(dev,
626 				       pkt->addr[i].addr,
627 				       pkt->addr[i].dma_length,
628 				       DMA_TO_DEVICE);
629 
630 		if (pkt->addr[i].kvaddr)
631 			kunmap(pkt->addr[i].page);
632 
633 		if (pkt->addr[i].put_page)
634 			put_page(pkt->addr[i].page);
635 		else
636 			__free_page(pkt->addr[i].page);
637 	} else if (pkt->addr[i].kvaddr) {
638 		/* for headers */
639 		if (pkt->addr[i].dma_mapped) {
640 			/* from kmalloc & dma mapped */
641 			dma_unmap_single(dev,
642 				       pkt->addr[i].addr,
643 				       pkt->addr[i].dma_length,
644 				       DMA_TO_DEVICE);
645 			kfree(pkt->addr[i].kvaddr);
646 		} else if (pkt->addr[i].addr) {
647 			/* free coherent mem from cache... */
648 			dma_pool_free(pq->header_cache,
649 			      pkt->addr[i].kvaddr, pkt->addr[i].addr);
650 		} else {
651 			/* from kmalloc but not dma mapped */
652 			kfree(pkt->addr[i].kvaddr);
653 		}
654 	}
655 }
656 
657 /* return number of pages pinned... */
658 static int qib_user_sdma_pin_pages(const struct qib_devdata *dd,
659 				   struct qib_user_sdma_queue *pq,
660 				   struct qib_user_sdma_pkt *pkt,
661 				   unsigned long addr, int tlen, int npages)
662 {
663 	struct page *pages[8];
664 	int i, j;
665 	int ret = 0;
666 
667 	while (npages) {
668 		if (npages > 8)
669 			j = 8;
670 		else
671 			j = npages;
672 
673 		ret = get_user_pages_fast(addr, j, FOLL_LONGTERM, pages);
674 		if (ret != j) {
675 			i = 0;
676 			j = ret;
677 			ret = -ENOMEM;
678 			goto free_pages;
679 		}
680 
681 		for (i = 0; i < j; i++) {
682 			/* map the pages... */
683 			unsigned long fofs = addr & ~PAGE_MASK;
684 			int flen = ((fofs + tlen) > PAGE_SIZE) ?
685 				(PAGE_SIZE - fofs) : tlen;
686 
687 			ret = qib_user_sdma_page_to_frags(dd, pq, pkt,
688 				pages[i], 1, fofs, flen, NULL);
689 			if (ret < 0) {
690 				/* current page has beed taken
691 				 * care of inside above call.
692 				 */
693 				i++;
694 				goto free_pages;
695 			}
696 
697 			addr += flen;
698 			tlen -= flen;
699 		}
700 
701 		npages -= j;
702 	}
703 
704 	goto done;
705 
706 	/* if error, return all pages not managed by pkt */
707 free_pages:
708 	while (i < j)
709 		put_page(pages[i++]);
710 
711 done:
712 	return ret;
713 }
714 
715 static int qib_user_sdma_pin_pkt(const struct qib_devdata *dd,
716 				 struct qib_user_sdma_queue *pq,
717 				 struct qib_user_sdma_pkt *pkt,
718 				 const struct iovec *iov,
719 				 unsigned long niov)
720 {
721 	int ret = 0;
722 	unsigned long idx;
723 
724 	for (idx = 0; idx < niov; idx++) {
725 		const int npages = qib_user_sdma_num_pages(iov + idx);
726 		const unsigned long addr = (unsigned long) iov[idx].iov_base;
727 
728 		ret = qib_user_sdma_pin_pages(dd, pq, pkt, addr,
729 					      iov[idx].iov_len, npages);
730 		if (ret < 0)
731 			goto free_pkt;
732 	}
733 
734 	goto done;
735 
736 free_pkt:
737 	/* we need to ignore the first entry here */
738 	for (idx = 1; idx < pkt->naddr; idx++)
739 		qib_user_sdma_free_pkt_frag(&dd->pcidev->dev, pq, pkt, idx);
740 
741 	/* need to dma unmap the first entry, this is to restore to
742 	 * the original state so that caller can free the memory in
743 	 * error condition. Caller does not know if dma mapped or not*/
744 	if (pkt->addr[0].dma_mapped) {
745 		dma_unmap_single(&dd->pcidev->dev,
746 		       pkt->addr[0].addr,
747 		       pkt->addr[0].dma_length,
748 		       DMA_TO_DEVICE);
749 		pkt->addr[0].addr = 0;
750 		pkt->addr[0].dma_mapped = 0;
751 	}
752 
753 done:
754 	return ret;
755 }
756 
757 static int qib_user_sdma_init_payload(const struct qib_devdata *dd,
758 				      struct qib_user_sdma_queue *pq,
759 				      struct qib_user_sdma_pkt *pkt,
760 				      const struct iovec *iov,
761 				      unsigned long niov, int npages)
762 {
763 	int ret = 0;
764 
765 	if (pkt->frag_size == pkt->bytes_togo &&
766 			npages >= ARRAY_SIZE(pkt->addr))
767 		ret = qib_user_sdma_coalesce(dd, pq, pkt, iov, niov);
768 	else
769 		ret = qib_user_sdma_pin_pkt(dd, pq, pkt, iov, niov);
770 
771 	return ret;
772 }
773 
774 /* free a packet list -- return counter value of last packet */
775 static void qib_user_sdma_free_pkt_list(struct device *dev,
776 					struct qib_user_sdma_queue *pq,
777 					struct list_head *list)
778 {
779 	struct qib_user_sdma_pkt *pkt, *pkt_next;
780 
781 	list_for_each_entry_safe(pkt, pkt_next, list, list) {
782 		int i;
783 
784 		for (i = 0; i < pkt->naddr; i++)
785 			qib_user_sdma_free_pkt_frag(dev, pq, pkt, i);
786 
787 		if (pkt->largepkt)
788 			kfree(pkt);
789 		else
790 			kmem_cache_free(pq->pkt_slab, pkt);
791 	}
792 	INIT_LIST_HEAD(list);
793 }
794 
795 /*
796  * copy headers, coalesce etc -- pq->lock must be held
797  *
798  * we queue all the packets to list, returning the
799  * number of bytes total.  list must be empty initially,
800  * as, if there is an error we clean it...
801  */
802 static int qib_user_sdma_queue_pkts(const struct qib_devdata *dd,
803 				    struct qib_pportdata *ppd,
804 				    struct qib_user_sdma_queue *pq,
805 				    const struct iovec *iov,
806 				    unsigned long niov,
807 				    struct list_head *list,
808 				    int *maxpkts, int *ndesc)
809 {
810 	unsigned long idx = 0;
811 	int ret = 0;
812 	int npkts = 0;
813 	__le32 *pbc;
814 	dma_addr_t dma_addr;
815 	struct qib_user_sdma_pkt *pkt = NULL;
816 	size_t len;
817 	size_t nw;
818 	u32 counter = pq->counter;
819 	u16 frag_size;
820 
821 	while (idx < niov && npkts < *maxpkts) {
822 		const unsigned long addr = (unsigned long) iov[idx].iov_base;
823 		const unsigned long idx_save = idx;
824 		unsigned pktnw;
825 		unsigned pktnwc;
826 		int nfrags = 0;
827 		int npages = 0;
828 		int bytes_togo = 0;
829 		int tiddma = 0;
830 		int cfur;
831 
832 		len = iov[idx].iov_len;
833 		nw = len >> 2;
834 
835 		if (len < QIB_USER_SDMA_MIN_HEADER_LENGTH ||
836 		    len > PAGE_SIZE || len & 3 || addr & 3) {
837 			ret = -EINVAL;
838 			goto free_list;
839 		}
840 
841 		pbc = qib_user_sdma_alloc_header(pq, len, &dma_addr);
842 		if (!pbc) {
843 			ret = -ENOMEM;
844 			goto free_list;
845 		}
846 
847 		cfur = copy_from_user(pbc, iov[idx].iov_base, len);
848 		if (cfur) {
849 			ret = -EFAULT;
850 			goto free_pbc;
851 		}
852 
853 		/*
854 		 * This assignment is a bit strange.  it's because the
855 		 * the pbc counts the number of 32 bit words in the full
856 		 * packet _except_ the first word of the pbc itself...
857 		 */
858 		pktnwc = nw - 1;
859 
860 		/*
861 		 * pktnw computation yields the number of 32 bit words
862 		 * that the caller has indicated in the PBC.  note that
863 		 * this is one less than the total number of words that
864 		 * goes to the send DMA engine as the first 32 bit word
865 		 * of the PBC itself is not counted.  Armed with this count,
866 		 * we can verify that the packet is consistent with the
867 		 * iovec lengths.
868 		 */
869 		pktnw = le32_to_cpu(*pbc) & 0xFFFF;
870 		if (pktnw < pktnwc) {
871 			ret = -EINVAL;
872 			goto free_pbc;
873 		}
874 
875 		idx++;
876 		while (pktnwc < pktnw && idx < niov) {
877 			const size_t slen = iov[idx].iov_len;
878 			const unsigned long faddr =
879 				(unsigned long) iov[idx].iov_base;
880 
881 			if (slen & 3 || faddr & 3 || !slen) {
882 				ret = -EINVAL;
883 				goto free_pbc;
884 			}
885 
886 			npages += qib_user_sdma_num_pages(&iov[idx]);
887 
888 			bytes_togo += slen;
889 			pktnwc += slen >> 2;
890 			idx++;
891 			nfrags++;
892 		}
893 
894 		if (pktnwc != pktnw) {
895 			ret = -EINVAL;
896 			goto free_pbc;
897 		}
898 
899 		frag_size = ((le32_to_cpu(*pbc))>>16) & 0xFFFF;
900 		if (((frag_size ? frag_size : bytes_togo) + len) >
901 						ppd->ibmaxlen) {
902 			ret = -EINVAL;
903 			goto free_pbc;
904 		}
905 
906 		if (frag_size) {
907 			int pktsize, tidsmsize, n;
908 
909 			n = npages*((2*PAGE_SIZE/frag_size)+1);
910 			pktsize = sizeof(*pkt) + sizeof(pkt->addr[0])*n;
911 
912 			/*
913 			 * Determine if this is tid-sdma or just sdma.
914 			 */
915 			tiddma = (((le32_to_cpu(pbc[7])>>
916 				QLOGIC_IB_I_TID_SHIFT)&
917 				QLOGIC_IB_I_TID_MASK) !=
918 				QLOGIC_IB_I_TID_MASK);
919 
920 			if (tiddma)
921 				tidsmsize = iov[idx].iov_len;
922 			else
923 				tidsmsize = 0;
924 
925 			pkt = kmalloc(pktsize+tidsmsize, GFP_KERNEL);
926 			if (!pkt) {
927 				ret = -ENOMEM;
928 				goto free_pbc;
929 			}
930 			pkt->largepkt = 1;
931 			pkt->frag_size = frag_size;
932 			pkt->addrlimit = n + ARRAY_SIZE(pkt->addr);
933 
934 			if (tiddma) {
935 				char *tidsm = (char *)pkt + pktsize;
936 
937 				cfur = copy_from_user(tidsm,
938 					iov[idx].iov_base, tidsmsize);
939 				if (cfur) {
940 					ret = -EFAULT;
941 					goto free_pkt;
942 				}
943 				pkt->tidsm =
944 					(struct qib_tid_session_member *)tidsm;
945 				pkt->tidsmcount = tidsmsize/
946 					sizeof(struct qib_tid_session_member);
947 				pkt->tidsmidx = 0;
948 				idx++;
949 			}
950 
951 			/*
952 			 * pbc 'fill1' field is borrowed to pass frag size,
953 			 * we need to clear it after picking frag size, the
954 			 * hardware requires this field to be zero.
955 			 */
956 			*pbc = cpu_to_le32(le32_to_cpu(*pbc) & 0x0000FFFF);
957 		} else {
958 			pkt = kmem_cache_alloc(pq->pkt_slab, GFP_KERNEL);
959 			if (!pkt) {
960 				ret = -ENOMEM;
961 				goto free_pbc;
962 			}
963 			pkt->largepkt = 0;
964 			pkt->frag_size = bytes_togo;
965 			pkt->addrlimit = ARRAY_SIZE(pkt->addr);
966 		}
967 		pkt->bytes_togo = bytes_togo;
968 		pkt->payload_size = 0;
969 		pkt->counter = counter;
970 		pkt->tiddma = tiddma;
971 
972 		/* setup the first header */
973 		qib_user_sdma_init_frag(pkt, 0, /* index */
974 			0, len,		/* offset, len */
975 			1, 0,		/* first last desc */
976 			0, 0,		/* put page, dma mapped */
977 			NULL, pbc,	/* struct page, virt addr */
978 			dma_addr, len);	/* dma addr, dma length */
979 		pkt->index = 0;
980 		pkt->naddr = 1;
981 
982 		if (nfrags) {
983 			ret = qib_user_sdma_init_payload(dd, pq, pkt,
984 							 iov + idx_save + 1,
985 							 nfrags, npages);
986 			if (ret < 0)
987 				goto free_pkt;
988 		} else {
989 			/* since there is no payload, mark the
990 			 * header as the last desc. */
991 			pkt->addr[0].last_desc = 1;
992 
993 			if (dma_addr == 0) {
994 				/*
995 				 * the header is not dma mapped yet.
996 				 * it should be from kmalloc.
997 				 */
998 				dma_addr = dma_map_single(&dd->pcidev->dev,
999 					pbc, len, DMA_TO_DEVICE);
1000 				if (dma_mapping_error(&dd->pcidev->dev,
1001 								dma_addr)) {
1002 					ret = -ENOMEM;
1003 					goto free_pkt;
1004 				}
1005 				pkt->addr[0].addr = dma_addr;
1006 				pkt->addr[0].dma_mapped = 1;
1007 			}
1008 		}
1009 
1010 		counter++;
1011 		npkts++;
1012 		pkt->pq = pq;
1013 		pkt->index = 0; /* reset index for push on hw */
1014 		*ndesc += pkt->naddr;
1015 
1016 		list_add_tail(&pkt->list, list);
1017 	}
1018 
1019 	*maxpkts = npkts;
1020 	ret = idx;
1021 	goto done;
1022 
1023 free_pkt:
1024 	if (pkt->largepkt)
1025 		kfree(pkt);
1026 	else
1027 		kmem_cache_free(pq->pkt_slab, pkt);
1028 free_pbc:
1029 	if (dma_addr)
1030 		dma_pool_free(pq->header_cache, pbc, dma_addr);
1031 	else
1032 		kfree(pbc);
1033 free_list:
1034 	qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, list);
1035 done:
1036 	return ret;
1037 }
1038 
1039 static void qib_user_sdma_set_complete_counter(struct qib_user_sdma_queue *pq,
1040 					       u32 c)
1041 {
1042 	pq->sent_counter = c;
1043 }
1044 
1045 /* try to clean out queue -- needs pq->lock */
1046 static int qib_user_sdma_queue_clean(struct qib_pportdata *ppd,
1047 				     struct qib_user_sdma_queue *pq)
1048 {
1049 	struct qib_devdata *dd = ppd->dd;
1050 	struct list_head free_list;
1051 	struct qib_user_sdma_pkt *pkt;
1052 	struct qib_user_sdma_pkt *pkt_prev;
1053 	unsigned long flags;
1054 	int ret = 0;
1055 
1056 	if (!pq->num_sending)
1057 		return 0;
1058 
1059 	INIT_LIST_HEAD(&free_list);
1060 
1061 	/*
1062 	 * We need this spin lock here because interrupt handler
1063 	 * might modify this list in qib_user_sdma_send_desc(), also
1064 	 * we can not get interrupted, otherwise it is a deadlock.
1065 	 */
1066 	spin_lock_irqsave(&pq->sent_lock, flags);
1067 	list_for_each_entry_safe(pkt, pkt_prev, &pq->sent, list) {
1068 		s64 descd = ppd->sdma_descq_removed - pkt->added;
1069 
1070 		if (descd < 0)
1071 			break;
1072 
1073 		list_move_tail(&pkt->list, &free_list);
1074 
1075 		/* one more packet cleaned */
1076 		ret++;
1077 		pq->num_sending--;
1078 	}
1079 	spin_unlock_irqrestore(&pq->sent_lock, flags);
1080 
1081 	if (!list_empty(&free_list)) {
1082 		u32 counter;
1083 
1084 		pkt = list_entry(free_list.prev,
1085 				 struct qib_user_sdma_pkt, list);
1086 		counter = pkt->counter;
1087 
1088 		qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &free_list);
1089 		qib_user_sdma_set_complete_counter(pq, counter);
1090 	}
1091 
1092 	return ret;
1093 }
1094 
1095 void qib_user_sdma_queue_destroy(struct qib_user_sdma_queue *pq)
1096 {
1097 	if (!pq)
1098 		return;
1099 
1100 	pq->sdma_rb_node->refcount--;
1101 	if (pq->sdma_rb_node->refcount == 0) {
1102 		rb_erase(&pq->sdma_rb_node->node, &qib_user_sdma_rb_root);
1103 		kfree(pq->sdma_rb_node);
1104 	}
1105 	dma_pool_destroy(pq->header_cache);
1106 	kmem_cache_destroy(pq->pkt_slab);
1107 	kfree(pq);
1108 }
1109 
1110 /* clean descriptor queue, returns > 0 if some elements cleaned */
1111 static int qib_user_sdma_hwqueue_clean(struct qib_pportdata *ppd)
1112 {
1113 	int ret;
1114 	unsigned long flags;
1115 
1116 	spin_lock_irqsave(&ppd->sdma_lock, flags);
1117 	ret = qib_sdma_make_progress(ppd);
1118 	spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1119 
1120 	return ret;
1121 }
1122 
1123 /* we're in close, drain packets so that we can cleanup successfully... */
1124 void qib_user_sdma_queue_drain(struct qib_pportdata *ppd,
1125 			       struct qib_user_sdma_queue *pq)
1126 {
1127 	struct qib_devdata *dd = ppd->dd;
1128 	unsigned long flags;
1129 	int i;
1130 
1131 	if (!pq)
1132 		return;
1133 
1134 	for (i = 0; i < QIB_USER_SDMA_DRAIN_TIMEOUT; i++) {
1135 		mutex_lock(&pq->lock);
1136 		if (!pq->num_pending && !pq->num_sending) {
1137 			mutex_unlock(&pq->lock);
1138 			break;
1139 		}
1140 		qib_user_sdma_hwqueue_clean(ppd);
1141 		qib_user_sdma_queue_clean(ppd, pq);
1142 		mutex_unlock(&pq->lock);
1143 		msleep(20);
1144 	}
1145 
1146 	if (pq->num_pending || pq->num_sending) {
1147 		struct qib_user_sdma_pkt *pkt;
1148 		struct qib_user_sdma_pkt *pkt_prev;
1149 		struct list_head free_list;
1150 
1151 		mutex_lock(&pq->lock);
1152 		spin_lock_irqsave(&ppd->sdma_lock, flags);
1153 		/*
1154 		 * Since we hold sdma_lock, it is safe without sent_lock.
1155 		 */
1156 		if (pq->num_pending) {
1157 			list_for_each_entry_safe(pkt, pkt_prev,
1158 					&ppd->sdma_userpending, list) {
1159 				if (pkt->pq == pq) {
1160 					list_move_tail(&pkt->list, &pq->sent);
1161 					pq->num_pending--;
1162 					pq->num_sending++;
1163 				}
1164 			}
1165 		}
1166 		spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1167 
1168 		qib_dev_err(dd, "user sdma lists not empty: forcing!\n");
1169 		INIT_LIST_HEAD(&free_list);
1170 		list_splice_init(&pq->sent, &free_list);
1171 		pq->num_sending = 0;
1172 		qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &free_list);
1173 		mutex_unlock(&pq->lock);
1174 	}
1175 }
1176 
1177 static inline __le64 qib_sdma_make_desc0(u8 gen,
1178 					 u64 addr, u64 dwlen, u64 dwoffset)
1179 {
1180 	return cpu_to_le64(/* SDmaPhyAddr[31:0] */
1181 			   ((addr & 0xfffffffcULL) << 32) |
1182 			   /* SDmaGeneration[1:0] */
1183 			   ((gen & 3ULL) << 30) |
1184 			   /* SDmaDwordCount[10:0] */
1185 			   ((dwlen & 0x7ffULL) << 16) |
1186 			   /* SDmaBufOffset[12:2] */
1187 			   (dwoffset & 0x7ffULL));
1188 }
1189 
1190 static inline __le64 qib_sdma_make_first_desc0(__le64 descq)
1191 {
1192 	return descq | cpu_to_le64(1ULL << 12);
1193 }
1194 
1195 static inline __le64 qib_sdma_make_last_desc0(__le64 descq)
1196 {
1197 					      /* last */  /* dma head */
1198 	return descq | cpu_to_le64(1ULL << 11 | 1ULL << 13);
1199 }
1200 
1201 static inline __le64 qib_sdma_make_desc1(u64 addr)
1202 {
1203 	/* SDmaPhyAddr[47:32] */
1204 	return cpu_to_le64(addr >> 32);
1205 }
1206 
1207 static void qib_user_sdma_send_frag(struct qib_pportdata *ppd,
1208 				    struct qib_user_sdma_pkt *pkt, int idx,
1209 				    unsigned ofs, u16 tail, u8 gen)
1210 {
1211 	const u64 addr = (u64) pkt->addr[idx].addr +
1212 		(u64) pkt->addr[idx].offset;
1213 	const u64 dwlen = (u64) pkt->addr[idx].length / 4;
1214 	__le64 *descqp;
1215 	__le64 descq0;
1216 
1217 	descqp = &ppd->sdma_descq[tail].qw[0];
1218 
1219 	descq0 = qib_sdma_make_desc0(gen, addr, dwlen, ofs);
1220 	if (pkt->addr[idx].first_desc)
1221 		descq0 = qib_sdma_make_first_desc0(descq0);
1222 	if (pkt->addr[idx].last_desc) {
1223 		descq0 = qib_sdma_make_last_desc0(descq0);
1224 		if (ppd->sdma_intrequest) {
1225 			descq0 |= cpu_to_le64(1ULL << 15);
1226 			ppd->sdma_intrequest = 0;
1227 		}
1228 	}
1229 
1230 	descqp[0] = descq0;
1231 	descqp[1] = qib_sdma_make_desc1(addr);
1232 }
1233 
1234 void qib_user_sdma_send_desc(struct qib_pportdata *ppd,
1235 				struct list_head *pktlist)
1236 {
1237 	struct qib_devdata *dd = ppd->dd;
1238 	u16 nfree, nsent;
1239 	u16 tail, tail_c;
1240 	u8 gen, gen_c;
1241 
1242 	nfree = qib_sdma_descq_freecnt(ppd);
1243 	if (!nfree)
1244 		return;
1245 
1246 retry:
1247 	nsent = 0;
1248 	tail_c = tail = ppd->sdma_descq_tail;
1249 	gen_c = gen = ppd->sdma_generation;
1250 	while (!list_empty(pktlist)) {
1251 		struct qib_user_sdma_pkt *pkt =
1252 			list_entry(pktlist->next, struct qib_user_sdma_pkt,
1253 				   list);
1254 		int i, j, c = 0;
1255 		unsigned ofs = 0;
1256 		u16 dtail = tail;
1257 
1258 		for (i = pkt->index; i < pkt->naddr && nfree; i++) {
1259 			qib_user_sdma_send_frag(ppd, pkt, i, ofs, tail, gen);
1260 			ofs += pkt->addr[i].length >> 2;
1261 
1262 			if (++tail == ppd->sdma_descq_cnt) {
1263 				tail = 0;
1264 				++gen;
1265 				ppd->sdma_intrequest = 1;
1266 			} else if (tail == (ppd->sdma_descq_cnt>>1)) {
1267 				ppd->sdma_intrequest = 1;
1268 			}
1269 			nfree--;
1270 			if (pkt->addr[i].last_desc == 0)
1271 				continue;
1272 
1273 			/*
1274 			 * If the packet is >= 2KB mtu equivalent, we
1275 			 * have to use the large buffers, and have to
1276 			 * mark each descriptor as part of a large
1277 			 * buffer packet.
1278 			 */
1279 			if (ofs > dd->piosize2kmax_dwords) {
1280 				for (j = pkt->index; j <= i; j++) {
1281 					ppd->sdma_descq[dtail].qw[0] |=
1282 						cpu_to_le64(1ULL << 14);
1283 					if (++dtail == ppd->sdma_descq_cnt)
1284 						dtail = 0;
1285 				}
1286 			}
1287 			c += i + 1 - pkt->index;
1288 			pkt->index = i + 1; /* index for next first */
1289 			tail_c = dtail = tail;
1290 			gen_c = gen;
1291 			ofs = 0;  /* reset for next packet */
1292 		}
1293 
1294 		ppd->sdma_descq_added += c;
1295 		nsent += c;
1296 		if (pkt->index == pkt->naddr) {
1297 			pkt->added = ppd->sdma_descq_added;
1298 			pkt->pq->added = pkt->added;
1299 			pkt->pq->num_pending--;
1300 			spin_lock(&pkt->pq->sent_lock);
1301 			pkt->pq->num_sending++;
1302 			list_move_tail(&pkt->list, &pkt->pq->sent);
1303 			spin_unlock(&pkt->pq->sent_lock);
1304 		}
1305 		if (!nfree || (nsent<<2) > ppd->sdma_descq_cnt)
1306 			break;
1307 	}
1308 
1309 	/* advance the tail on the chip if necessary */
1310 	if (ppd->sdma_descq_tail != tail_c) {
1311 		ppd->sdma_generation = gen_c;
1312 		dd->f_sdma_update_tail(ppd, tail_c);
1313 	}
1314 
1315 	if (nfree && !list_empty(pktlist))
1316 		goto retry;
1317 }
1318 
1319 /* pq->lock must be held, get packets on the wire... */
1320 static int qib_user_sdma_push_pkts(struct qib_pportdata *ppd,
1321 				 struct qib_user_sdma_queue *pq,
1322 				 struct list_head *pktlist, int count)
1323 {
1324 	unsigned long flags;
1325 
1326 	if (unlikely(!(ppd->lflags & QIBL_LINKACTIVE)))
1327 		return -ECOMM;
1328 
1329 	/* non-blocking mode */
1330 	if (pq->sdma_rb_node->refcount > 1) {
1331 		spin_lock_irqsave(&ppd->sdma_lock, flags);
1332 		if (unlikely(!__qib_sdma_running(ppd))) {
1333 			spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1334 			return -ECOMM;
1335 		}
1336 		pq->num_pending += count;
1337 		list_splice_tail_init(pktlist, &ppd->sdma_userpending);
1338 		qib_user_sdma_send_desc(ppd, &ppd->sdma_userpending);
1339 		spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1340 		return 0;
1341 	}
1342 
1343 	/* In this case, descriptors from this process are not
1344 	 * linked to ppd pending queue, interrupt handler
1345 	 * won't update this process, it is OK to directly
1346 	 * modify without sdma lock.
1347 	 */
1348 
1349 
1350 	pq->num_pending += count;
1351 	/*
1352 	 * Blocking mode for single rail process, we must
1353 	 * release/regain sdma_lock to give other process
1354 	 * chance to make progress. This is important for
1355 	 * performance.
1356 	 */
1357 	do {
1358 		spin_lock_irqsave(&ppd->sdma_lock, flags);
1359 		if (unlikely(!__qib_sdma_running(ppd))) {
1360 			spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1361 			return -ECOMM;
1362 		}
1363 		qib_user_sdma_send_desc(ppd, pktlist);
1364 		if (!list_empty(pktlist))
1365 			qib_sdma_make_progress(ppd);
1366 		spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1367 	} while (!list_empty(pktlist));
1368 
1369 	return 0;
1370 }
1371 
1372 int qib_user_sdma_writev(struct qib_ctxtdata *rcd,
1373 			 struct qib_user_sdma_queue *pq,
1374 			 const struct iovec *iov,
1375 			 unsigned long dim)
1376 {
1377 	struct qib_devdata *dd = rcd->dd;
1378 	struct qib_pportdata *ppd = rcd->ppd;
1379 	int ret = 0;
1380 	struct list_head list;
1381 	int npkts = 0;
1382 
1383 	INIT_LIST_HEAD(&list);
1384 
1385 	mutex_lock(&pq->lock);
1386 
1387 	/* why not -ECOMM like qib_user_sdma_push_pkts() below? */
1388 	if (!qib_sdma_running(ppd))
1389 		goto done_unlock;
1390 
1391 	/* if I have packets not complete yet */
1392 	if (pq->added > ppd->sdma_descq_removed)
1393 		qib_user_sdma_hwqueue_clean(ppd);
1394 	/* if I have complete packets to be freed */
1395 	if (pq->num_sending)
1396 		qib_user_sdma_queue_clean(ppd, pq);
1397 
1398 	while (dim) {
1399 		int mxp = 1;
1400 		int ndesc = 0;
1401 
1402 		ret = qib_user_sdma_queue_pkts(dd, ppd, pq,
1403 				iov, dim, &list, &mxp, &ndesc);
1404 		if (ret < 0)
1405 			goto done_unlock;
1406 		else {
1407 			dim -= ret;
1408 			iov += ret;
1409 		}
1410 
1411 		/* force packets onto the sdma hw queue... */
1412 		if (!list_empty(&list)) {
1413 			/*
1414 			 * Lazily clean hw queue.
1415 			 */
1416 			if (qib_sdma_descq_freecnt(ppd) < ndesc) {
1417 				qib_user_sdma_hwqueue_clean(ppd);
1418 				if (pq->num_sending)
1419 					qib_user_sdma_queue_clean(ppd, pq);
1420 			}
1421 
1422 			ret = qib_user_sdma_push_pkts(ppd, pq, &list, mxp);
1423 			if (ret < 0)
1424 				goto done_unlock;
1425 			else {
1426 				npkts += mxp;
1427 				pq->counter += mxp;
1428 			}
1429 		}
1430 	}
1431 
1432 done_unlock:
1433 	if (!list_empty(&list))
1434 		qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &list);
1435 	mutex_unlock(&pq->lock);
1436 
1437 	return (ret < 0) ? ret : npkts;
1438 }
1439 
1440 int qib_user_sdma_make_progress(struct qib_pportdata *ppd,
1441 				struct qib_user_sdma_queue *pq)
1442 {
1443 	int ret = 0;
1444 
1445 	mutex_lock(&pq->lock);
1446 	qib_user_sdma_hwqueue_clean(ppd);
1447 	ret = qib_user_sdma_queue_clean(ppd, pq);
1448 	mutex_unlock(&pq->lock);
1449 
1450 	return ret;
1451 }
1452 
1453 u32 qib_user_sdma_complete_counter(const struct qib_user_sdma_queue *pq)
1454 {
1455 	return pq ? pq->sent_counter : 0;
1456 }
1457 
1458 u32 qib_user_sdma_inflight_counter(struct qib_user_sdma_queue *pq)
1459 {
1460 	return pq ? pq->counter : 0;
1461 }
1462