xref: /openbmc/linux/drivers/misc/genwqe/card_ddcb.c (revision 0e17c50f)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /**
3  * IBM Accelerator Family 'GenWQE'
4  *
5  * (C) Copyright IBM Corp. 2013
6  *
7  * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
8  * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
9  * Author: Michael Jung <mijung@gmx.net>
10  * Author: Michael Ruettger <michael@ibmra.de>
11  */
12 
13 /*
14  * Device Driver Control Block (DDCB) queue support. Definition of
15  * interrupt handlers for queue support as well as triggering the
16  * health monitor code in case of problems. The current hardware uses
17  * an MSI interrupt which is shared between error handling and
18  * functional code.
19  */
20 
21 #include <linux/types.h>
22 #include <linux/sched.h>
23 #include <linux/wait.h>
24 #include <linux/pci.h>
25 #include <linux/string.h>
26 #include <linux/dma-mapping.h>
27 #include <linux/delay.h>
28 #include <linux/module.h>
29 #include <linux/interrupt.h>
30 #include <linux/crc-itu-t.h>
31 
32 #include "card_base.h"
33 #include "card_ddcb.h"
34 
35 /*
36  * N: next DDCB, this is where the next DDCB will be put.
37  * A: active DDCB, this is where the code will look for the next completion.
38  * x: DDCB is enqueued, we are waiting for its completion.
39 
40  * Situation (1): Empty queue
41  *  +---+---+---+---+---+---+---+---+
42  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
43  *  |   |   |   |   |   |   |   |   |
44  *  +---+---+---+---+---+---+---+---+
45  *           A/N
46  *  enqueued_ddcbs = A - N = 2 - 2 = 0
47  *
48  * Situation (2): Wrapped, N > A
49  *  +---+---+---+---+---+---+---+---+
50  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
51  *  |   |   | x | x |   |   |   |   |
52  *  +---+---+---+---+---+---+---+---+
53  *            A       N
54  *  enqueued_ddcbs = N - A = 4 - 2 = 2
55  *
56  * Situation (3): Queue wrapped, A > N
57  *  +---+---+---+---+---+---+---+---+
58  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
59  *  | x | x |   |   | x | x | x | x |
60  *  +---+---+---+---+---+---+---+---+
61  *            N       A
62  *  enqueued_ddcbs = queue_max  - (A - N) = 8 - (4 - 2) = 6
63  *
64  * Situation (4a): Queue full N > A
65  *  +---+---+---+---+---+---+---+---+
66  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
67  *  | x | x | x | x | x | x | x |   |
68  *  +---+---+---+---+---+---+---+---+
69  *    A                           N
70  *
71  *  enqueued_ddcbs = N - A = 7 - 0 = 7
72  *
73  * Situation (4a): Queue full A > N
74  *  +---+---+---+---+---+---+---+---+
75  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
76  *  | x | x | x |   | x | x | x | x |
77  *  +---+---+---+---+---+---+---+---+
78  *                N   A
79  *  enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7
80  */
81 
82 static int queue_empty(struct ddcb_queue *queue)
83 {
84 	return queue->ddcb_next == queue->ddcb_act;
85 }
86 
87 static int queue_enqueued_ddcbs(struct ddcb_queue *queue)
88 {
89 	if (queue->ddcb_next >= queue->ddcb_act)
90 		return queue->ddcb_next - queue->ddcb_act;
91 
92 	return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next);
93 }
94 
95 static int queue_free_ddcbs(struct ddcb_queue *queue)
96 {
97 	int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1;
98 
99 	if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */
100 		return 0;
101 	}
102 	return free_ddcbs;
103 }
104 
105 /*
106  * Use of the PRIV field in the DDCB for queue debugging:
107  *
108  * (1) Trying to get rid of a DDCB which saw a timeout:
109  *     pddcb->priv[6] = 0xcc;   # cleared
110  *
111  * (2) Append a DDCB via NEXT bit:
112  *     pddcb->priv[7] = 0xaa;	# appended
113  *
114  * (3) DDCB needed tapping:
115  *     pddcb->priv[7] = 0xbb;   # tapped
116  *
117  * (4) DDCB marked as correctly finished:
118  *     pddcb->priv[6] = 0xff;	# finished
119  */
120 
121 static inline void ddcb_mark_tapped(struct ddcb *pddcb)
122 {
123 	pddcb->priv[7] = 0xbb;  /* tapped */
124 }
125 
126 static inline void ddcb_mark_appended(struct ddcb *pddcb)
127 {
128 	pddcb->priv[7] = 0xaa;	/* appended */
129 }
130 
131 static inline void ddcb_mark_cleared(struct ddcb *pddcb)
132 {
133 	pddcb->priv[6] = 0xcc; /* cleared */
134 }
135 
136 static inline void ddcb_mark_finished(struct ddcb *pddcb)
137 {
138 	pddcb->priv[6] = 0xff;	/* finished */
139 }
140 
141 static inline void ddcb_mark_unused(struct ddcb *pddcb)
142 {
143 	pddcb->priv_64 = cpu_to_be64(0); /* not tapped */
144 }
145 
146 /**
147  * genwqe_crc16() - Generate 16-bit crc as required for DDCBs
148  * @buff:       pointer to data buffer
149  * @len:        length of data for calculation
150  * @init:       initial crc (0xffff at start)
151  *
152  * Polynomial = x^16 + x^12 + x^5 + 1   (0x1021)
153  * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff
154  *          should result in a crc16 of 0x89c3
155  *
156  * Return: crc16 checksum in big endian format !
157  */
158 static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init)
159 {
160 	return crc_itu_t(init, buff, len);
161 }
162 
163 static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue)
164 {
165 	int i;
166 	struct ddcb *pddcb;
167 	unsigned long flags;
168 	struct pci_dev *pci_dev = cd->pci_dev;
169 
170 	spin_lock_irqsave(&cd->print_lock, flags);
171 
172 	dev_info(&pci_dev->dev,
173 		 "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n",
174 		 cd->card_idx, queue->ddcb_act, queue->ddcb_next);
175 
176 	pddcb = queue->ddcb_vaddr;
177 	for (i = 0; i < queue->ddcb_max; i++) {
178 		dev_err(&pci_dev->dev,
179 			"  %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n",
180 			i == queue->ddcb_act ? '>' : ' ',
181 			i,
182 			be16_to_cpu(pddcb->retc_16),
183 			be16_to_cpu(pddcb->seqnum_16),
184 			pddcb->hsi,
185 			pddcb->shi,
186 			be64_to_cpu(pddcb->priv_64),
187 			pddcb->cmd);
188 		pddcb++;
189 	}
190 	spin_unlock_irqrestore(&cd->print_lock, flags);
191 }
192 
193 struct genwqe_ddcb_cmd *ddcb_requ_alloc(void)
194 {
195 	struct ddcb_requ *req;
196 
197 	req = kzalloc(sizeof(*req), GFP_KERNEL);
198 	if (!req)
199 		return NULL;
200 
201 	return &req->cmd;
202 }
203 
204 void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd)
205 {
206 	struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
207 
208 	kfree(req);
209 }
210 
211 static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req)
212 {
213 	return req->req_state;
214 }
215 
216 static inline void ddcb_requ_set_state(struct ddcb_requ *req,
217 				       enum genwqe_requ_state new_state)
218 {
219 	req->req_state = new_state;
220 }
221 
222 static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req)
223 {
224 	return req->cmd.ddata_addr != 0x0;
225 }
226 
227 /**
228  * ddcb_requ_finished() - Returns the hardware state of the associated DDCB
229  * @cd:          pointer to genwqe device descriptor
230  * @req:         DDCB work request
231  *
232  * Status of ddcb_requ mirrors this hardware state, but is copied in
233  * the ddcb_requ on interrupt/polling function. The lowlevel code
234  * should check the hardware state directly, the higher level code
235  * should check the copy.
236  *
237  * This function will also return true if the state of the queue is
238  * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the
239  * shutdown case.
240  */
241 static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req)
242 {
243 	return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) ||
244 		(cd->card_state != GENWQE_CARD_USED);
245 }
246 
247 /**
248  * enqueue_ddcb() - Enqueue a DDCB
249  * @cd:         pointer to genwqe device descriptor
250  * @queue:	queue this operation should be done on
251  * @ddcb_no:    pointer to ddcb number being tapped
252  *
253  * Start execution of DDCB by tapping or append to queue via NEXT
254  * bit. This is done by an atomic 'compare and swap' instruction and
255  * checking SHI and HSI of the previous DDCB.
256  *
257  * This function must only be called with ddcb_lock held.
258  *
259  * Return: 1 if new DDCB is appended to previous
260  *         2 if DDCB queue is tapped via register/simulation
261  */
262 #define RET_DDCB_APPENDED 1
263 #define RET_DDCB_TAPPED   2
264 
265 static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue,
266 			struct ddcb *pddcb, int ddcb_no)
267 {
268 	unsigned int try;
269 	int prev_no;
270 	struct ddcb *prev_ddcb;
271 	__be32 old, new, icrc_hsi_shi;
272 	u64 num;
273 
274 	/*
275 	 * For performance checks a Dispatch Timestamp can be put into
276 	 * DDCB It is supposed to use the SLU's free running counter,
277 	 * but this requires PCIe cycles.
278 	 */
279 	ddcb_mark_unused(pddcb);
280 
281 	/* check previous DDCB if already fetched */
282 	prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1;
283 	prev_ddcb = &queue->ddcb_vaddr[prev_no];
284 
285 	/*
286 	 * It might have happened that the HSI.FETCHED bit is
287 	 * set. Retry in this case. Therefore I expect maximum 2 times
288 	 * trying.
289 	 */
290 	ddcb_mark_appended(pddcb);
291 	for (try = 0; try < 2; try++) {
292 		old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */
293 
294 		/* try to append via NEXT bit if prev DDCB is not completed */
295 		if ((old & DDCB_COMPLETED_BE32) != 0x00000000)
296 			break;
297 
298 		new = (old | DDCB_NEXT_BE32);
299 
300 		wmb();		/* need to ensure write ordering */
301 		icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new);
302 
303 		if (icrc_hsi_shi == old)
304 			return RET_DDCB_APPENDED; /* appended to queue */
305 	}
306 
307 	/* Queue must be re-started by updating QUEUE_OFFSET */
308 	ddcb_mark_tapped(pddcb);
309 	num = (u64)ddcb_no << 8;
310 
311 	wmb();			/* need to ensure write ordering */
312 	__genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */
313 
314 	return RET_DDCB_TAPPED;
315 }
316 
317 /**
318  * copy_ddcb_results() - Copy output state from real DDCB to request
319  *
320  * Copy DDCB ASV to request struct. There is no endian
321  * conversion made, since data structure in ASV is still
322  * unknown here.
323  *
324  * This is needed by:
325  *   - genwqe_purge_ddcb()
326  *   - genwqe_check_ddcb_queue()
327  */
328 static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no)
329 {
330 	struct ddcb_queue *queue = req->queue;
331 	struct ddcb *pddcb = &queue->ddcb_vaddr[req->num];
332 
333 	memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH);
334 
335 	/* copy status flags of the variant part */
336 	req->cmd.vcrc     = be16_to_cpu(pddcb->vcrc_16);
337 	req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64);
338 	req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64);
339 
340 	req->cmd.attn     = be16_to_cpu(pddcb->attn_16);
341 	req->cmd.progress = be32_to_cpu(pddcb->progress_32);
342 	req->cmd.retc     = be16_to_cpu(pddcb->retc_16);
343 
344 	if (ddcb_requ_collect_debug_data(req)) {
345 		int prev_no = (ddcb_no == 0) ?
346 			queue->ddcb_max - 1 : ddcb_no - 1;
347 		struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no];
348 
349 		memcpy(&req->debug_data.ddcb_finished, pddcb,
350 		       sizeof(req->debug_data.ddcb_finished));
351 		memcpy(&req->debug_data.ddcb_prev, prev_pddcb,
352 		       sizeof(req->debug_data.ddcb_prev));
353 	}
354 }
355 
356 /**
357  * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work equests.
358  * @cd:         pointer to genwqe device descriptor
359  *
360  * Return: Number of DDCBs which were finished
361  */
362 static int genwqe_check_ddcb_queue(struct genwqe_dev *cd,
363 				   struct ddcb_queue *queue)
364 {
365 	unsigned long flags;
366 	int ddcbs_finished = 0;
367 	struct pci_dev *pci_dev = cd->pci_dev;
368 
369 	spin_lock_irqsave(&queue->ddcb_lock, flags);
370 
371 	/* FIXME avoid soft locking CPU */
372 	while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) {
373 
374 		struct ddcb *pddcb;
375 		struct ddcb_requ *req;
376 		u16 vcrc, vcrc_16, retc_16;
377 
378 		pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
379 
380 		if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) ==
381 		    0x00000000)
382 			goto go_home; /* not completed, continue waiting */
383 
384 		wmb();  /*  Add sync to decouple prev. read operations */
385 
386 		/* Note: DDCB could be purged */
387 		req = queue->ddcb_req[queue->ddcb_act];
388 		if (req == NULL) {
389 			/* this occurs if DDCB is purged, not an error */
390 			/* Move active DDCB further; Nothing to do anymore. */
391 			goto pick_next_one;
392 		}
393 
394 		/*
395 		 * HSI=0x44 (fetched and completed), but RETC is
396 		 * 0x101, or even worse 0x000.
397 		 *
398 		 * In case of seeing the queue in inconsistent state
399 		 * we read the errcnts and the queue status to provide
400 		 * a trigger for our PCIe analyzer stop capturing.
401 		 */
402 		retc_16 = be16_to_cpu(pddcb->retc_16);
403 		if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) {
404 			u64 errcnts, status;
405 			u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr;
406 
407 			errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS);
408 			status  = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
409 
410 			dev_err(&pci_dev->dev,
411 				"[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n",
412 				__func__, be16_to_cpu(pddcb->seqnum_16),
413 				pddcb->hsi, retc_16, errcnts, status,
414 				queue->ddcb_daddr + ddcb_offs);
415 		}
416 
417 		copy_ddcb_results(req, queue->ddcb_act);
418 		queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */
419 
420 		dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num);
421 		genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
422 
423 		ddcb_mark_finished(pddcb);
424 
425 		/* calculate CRC_16 to see if VCRC is correct */
426 		vcrc = genwqe_crc16(pddcb->asv,
427 				   VCRC_LENGTH(req->cmd.asv_length),
428 				   0xffff);
429 		vcrc_16 = be16_to_cpu(pddcb->vcrc_16);
430 		if (vcrc != vcrc_16) {
431 			printk_ratelimited(KERN_ERR
432 				"%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n",
433 				GENWQE_DEVNAME, dev_name(&pci_dev->dev),
434 				pddcb->pre, VCRC_LENGTH(req->cmd.asv_length),
435 				vcrc, vcrc_16);
436 		}
437 
438 		ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
439 		queue->ddcbs_completed++;
440 		queue->ddcbs_in_flight--;
441 
442 		/* wake up process waiting for this DDCB, and
443                    processes on the busy queue */
444 		wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
445 		wake_up_interruptible(&queue->busy_waitq);
446 
447 pick_next_one:
448 		queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max;
449 		ddcbs_finished++;
450 	}
451 
452  go_home:
453 	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
454 	return ddcbs_finished;
455 }
456 
457 /**
458  * __genwqe_wait_ddcb(): Waits until DDCB is completed
459  * @cd:         pointer to genwqe device descriptor
460  * @req:        pointer to requsted DDCB parameters
461  *
462  * The Service Layer will update the RETC in DDCB when processing is
463  * pending or done.
464  *
465  * Return: > 0 remaining jiffies, DDCB completed
466  *           -ETIMEDOUT	when timeout
467  *           -ERESTARTSYS when ^C
468  *           -EINVAL when unknown error condition
469  *
470  * When an error is returned the called needs to ensure that
471  * purge_ddcb() is being called to get the &req removed from the
472  * queue.
473  */
474 int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
475 {
476 	int rc;
477 	unsigned int ddcb_no;
478 	struct ddcb_queue *queue;
479 	struct pci_dev *pci_dev = cd->pci_dev;
480 
481 	if (req == NULL)
482 		return -EINVAL;
483 
484 	queue = req->queue;
485 	if (queue == NULL)
486 		return -EINVAL;
487 
488 	ddcb_no = req->num;
489 	if (ddcb_no >= queue->ddcb_max)
490 		return -EINVAL;
491 
492 	rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no],
493 				ddcb_requ_finished(cd, req),
494 				GENWQE_DDCB_SOFTWARE_TIMEOUT * HZ);
495 
496 	/*
497 	 * We need to distinguish 3 cases here:
498 	 *   1. rc == 0              timeout occured
499 	 *   2. rc == -ERESTARTSYS   signal received
500 	 *   3. rc > 0               remaining jiffies condition is true
501 	 */
502 	if (rc == 0) {
503 		struct ddcb_queue *queue = req->queue;
504 		struct ddcb *pddcb;
505 
506 		/*
507 		 * Timeout may be caused by long task switching time.
508 		 * When timeout happens, check if the request has
509 		 * meanwhile completed.
510 		 */
511 		genwqe_check_ddcb_queue(cd, req->queue);
512 		if (ddcb_requ_finished(cd, req))
513 			return rc;
514 
515 		dev_err(&pci_dev->dev,
516 			"[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n",
517 			__func__, req->num, rc,	ddcb_requ_get_state(req),
518 			req);
519 		dev_err(&pci_dev->dev,
520 			"[%s]      IO_QUEUE_STATUS=0x%016llx\n", __func__,
521 			__genwqe_readq(cd, queue->IO_QUEUE_STATUS));
522 
523 		pddcb = &queue->ddcb_vaddr[req->num];
524 		genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
525 
526 		print_ddcb_info(cd, req->queue);
527 		return -ETIMEDOUT;
528 
529 	} else if (rc == -ERESTARTSYS) {
530 		return rc;
531 		/*
532 		 * EINTR:       Stops the application
533 		 * ERESTARTSYS: Restartable systemcall; called again
534 		 */
535 
536 	} else if (rc < 0) {
537 		dev_err(&pci_dev->dev,
538 			"[%s] err: DDCB#%d unknown result (rc=%d) %d!\n",
539 			__func__, req->num, rc, ddcb_requ_get_state(req));
540 		return -EINVAL;
541 	}
542 
543 	/* Severe error occured. Driver is forced to stop operation */
544 	if (cd->card_state != GENWQE_CARD_USED) {
545 		dev_err(&pci_dev->dev,
546 			"[%s] err: DDCB#%d forced to stop (rc=%d)\n",
547 			__func__, req->num, rc);
548 		return -EIO;
549 	}
550 	return rc;
551 }
552 
553 /**
554  * get_next_ddcb() - Get next available DDCB
555  * @cd:         pointer to genwqe device descriptor
556  *
557  * DDCB's content is completely cleared but presets for PRE and
558  * SEQNUM. This function must only be called when ddcb_lock is held.
559  *
560  * Return: NULL if no empty DDCB available otherwise ptr to next DDCB.
561  */
562 static struct ddcb *get_next_ddcb(struct genwqe_dev *cd,
563 				  struct ddcb_queue *queue,
564 				  int *num)
565 {
566 	u64 *pu64;
567 	struct ddcb *pddcb;
568 
569 	if (queue_free_ddcbs(queue) == 0) /* queue is  full */
570 		return NULL;
571 
572 	/* find new ddcb */
573 	pddcb = &queue->ddcb_vaddr[queue->ddcb_next];
574 
575 	/* if it is not completed, we are not allowed to use it */
576 	/* barrier(); */
577 	if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000)
578 		return NULL;
579 
580 	*num = queue->ddcb_next;	/* internal DDCB number */
581 	queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max;
582 
583 	/* clear important DDCB fields */
584 	pu64 = (u64 *)pddcb;
585 	pu64[0] = 0ULL;		/* offs 0x00 (ICRC,HSI,SHI,...) */
586 	pu64[1] = 0ULL;		/* offs 0x01 (ACFUNC,CMD...) */
587 
588 	/* destroy previous results in ASV */
589 	pu64[0x80/8] = 0ULL;	/* offs 0x80 (ASV + 0) */
590 	pu64[0x88/8] = 0ULL;	/* offs 0x88 (ASV + 0x08) */
591 	pu64[0x90/8] = 0ULL;	/* offs 0x90 (ASV + 0x10) */
592 	pu64[0x98/8] = 0ULL;	/* offs 0x98 (ASV + 0x18) */
593 	pu64[0xd0/8] = 0ULL;	/* offs 0xd0 (RETC,ATTN...) */
594 
595 	pddcb->pre = DDCB_PRESET_PRE; /* 128 */
596 	pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++);
597 	return pddcb;
598 }
599 
600 /**
601  * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue
602  * @cd:         genwqe device descriptor
603  * @req:        DDCB request
604  *
605  * This will fail when the request was already FETCHED. In this case
606  * we need to wait until it is finished. Else the DDCB can be
607  * reused. This function also ensures that the request data structure
608  * is removed from ddcb_req[].
609  *
610  * Do not forget to call this function when genwqe_wait_ddcb() fails,
611  * such that the request gets really removed from ddcb_req[].
612  *
613  * Return: 0 success
614  */
615 int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
616 {
617 	struct ddcb *pddcb = NULL;
618 	unsigned int t;
619 	unsigned long flags;
620 	struct ddcb_queue *queue = req->queue;
621 	struct pci_dev *pci_dev = cd->pci_dev;
622 	u64 queue_status;
623 	__be32 icrc_hsi_shi = 0x0000;
624 	__be32 old, new;
625 
626 	/* unsigned long flags; */
627 	if (GENWQE_DDCB_SOFTWARE_TIMEOUT <= 0) {
628 		dev_err(&pci_dev->dev,
629 			"[%s] err: software timeout is not set!\n", __func__);
630 		return -EFAULT;
631 	}
632 
633 	pddcb = &queue->ddcb_vaddr[req->num];
634 
635 	for (t = 0; t < GENWQE_DDCB_SOFTWARE_TIMEOUT * 10; t++) {
636 
637 		spin_lock_irqsave(&queue->ddcb_lock, flags);
638 
639 		/* Check if req was meanwhile finished */
640 		if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED)
641 			goto go_home;
642 
643 		/* try to set PURGE bit if FETCHED/COMPLETED are not set */
644 		old = pddcb->icrc_hsi_shi_32;	/* read SHI/HSI in BE32 */
645 		if ((old & DDCB_FETCHED_BE32) == 0x00000000) {
646 
647 			new = (old | DDCB_PURGE_BE32);
648 			icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32,
649 					       old, new);
650 			if (icrc_hsi_shi == old)
651 				goto finish_ddcb;
652 		}
653 
654 		/* normal finish with HSI bit */
655 		barrier();
656 		icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
657 		if (icrc_hsi_shi & DDCB_COMPLETED_BE32)
658 			goto finish_ddcb;
659 
660 		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
661 
662 		/*
663 		 * Here the check_ddcb() function will most likely
664 		 * discover this DDCB to be finished some point in
665 		 * time. It will mark the req finished and free it up
666 		 * in the list.
667 		 */
668 
669 		copy_ddcb_results(req, req->num); /* for the failing case */
670 		msleep(100); /* sleep for 1/10 second and try again */
671 		continue;
672 
673 finish_ddcb:
674 		copy_ddcb_results(req, req->num);
675 		ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
676 		queue->ddcbs_in_flight--;
677 		queue->ddcb_req[req->num] = NULL; /* delete from array */
678 		ddcb_mark_cleared(pddcb);
679 
680 		/* Move active DDCB further; Nothing to do here anymore. */
681 
682 		/*
683 		 * We need to ensure that there is at least one free
684 		 * DDCB in the queue. To do that, we must update
685 		 * ddcb_act only if the COMPLETED bit is set for the
686 		 * DDCB we are working on else we treat that DDCB even
687 		 * if we PURGED it as occupied (hardware is supposed
688 		 * to set the COMPLETED bit yet!).
689 		 */
690 		icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
691 		if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) &&
692 		    (queue->ddcb_act == req->num)) {
693 			queue->ddcb_act = ((queue->ddcb_act + 1) %
694 					   queue->ddcb_max);
695 		}
696 go_home:
697 		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
698 		return 0;
699 	}
700 
701 	/*
702 	 * If the card is dead and the queue is forced to stop, we
703 	 * might see this in the queue status register.
704 	 */
705 	queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
706 
707 	dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num);
708 	genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
709 
710 	dev_err(&pci_dev->dev,
711 		"[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n",
712 		__func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT,
713 		queue_status);
714 
715 	print_ddcb_info(cd, req->queue);
716 
717 	return -EFAULT;
718 }
719 
720 int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d)
721 {
722 	int len;
723 	struct pci_dev *pci_dev = cd->pci_dev;
724 
725 	if (d == NULL) {
726 		dev_err(&pci_dev->dev,
727 			"[%s] err: invalid memory for debug data!\n",
728 			__func__);
729 		return -EFAULT;
730 	}
731 
732 	len  = sizeof(d->driver_version);
733 	snprintf(d->driver_version, len, "%s", DRV_VERSION);
734 	d->slu_unitcfg = cd->slu_unitcfg;
735 	d->app_unitcfg = cd->app_unitcfg;
736 	return 0;
737 }
738 
739 /**
740  * __genwqe_enqueue_ddcb() - Enqueue a DDCB
741  * @cd:         pointer to genwqe device descriptor
742  * @req:        pointer to DDCB execution request
743  * @f_flags:    file mode: blocking, non-blocking
744  *
745  * Return: 0 if enqueuing succeeded
746  *         -EIO if card is unusable/PCIe problems
747  *         -EBUSY if enqueuing failed
748  */
749 int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req,
750 			  unsigned int f_flags)
751 {
752 	struct ddcb *pddcb;
753 	unsigned long flags;
754 	struct ddcb_queue *queue;
755 	struct pci_dev *pci_dev = cd->pci_dev;
756 	u16 icrc;
757 
758  retry:
759 	if (cd->card_state != GENWQE_CARD_USED) {
760 		printk_ratelimited(KERN_ERR
761 			"%s %s: [%s] Card is unusable/PCIe problem Req#%d\n",
762 			GENWQE_DEVNAME, dev_name(&pci_dev->dev),
763 			__func__, req->num);
764 		return -EIO;
765 	}
766 
767 	queue = req->queue = &cd->queue;
768 
769 	/* FIXME circumvention to improve performance when no irq is
770 	 * there.
771 	 */
772 	if (GENWQE_POLLING_ENABLED)
773 		genwqe_check_ddcb_queue(cd, queue);
774 
775 	/*
776 	 * It must be ensured to process all DDCBs in successive
777 	 * order. Use a lock here in order to prevent nested DDCB
778 	 * enqueuing.
779 	 */
780 	spin_lock_irqsave(&queue->ddcb_lock, flags);
781 
782 	pddcb = get_next_ddcb(cd, queue, &req->num);	/* get ptr and num */
783 	if (pddcb == NULL) {
784 		int rc;
785 
786 		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
787 
788 		if (f_flags & O_NONBLOCK) {
789 			queue->return_on_busy++;
790 			return -EBUSY;
791 		}
792 
793 		queue->wait_on_busy++;
794 		rc = wait_event_interruptible(queue->busy_waitq,
795 					      queue_free_ddcbs(queue) != 0);
796 		dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n",
797 			__func__, rc);
798 		if (rc == -ERESTARTSYS)
799 			return rc;  /* interrupted by a signal */
800 
801 		goto retry;
802 	}
803 
804 	if (queue->ddcb_req[req->num] != NULL) {
805 		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
806 
807 		dev_err(&pci_dev->dev,
808 			"[%s] picked DDCB %d with req=%p still in use!!\n",
809 			__func__, req->num, req);
810 		return -EFAULT;
811 	}
812 	ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED);
813 	queue->ddcb_req[req->num] = req;
814 
815 	pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts);
816 	pddcb->cmd = req->cmd.cmd;
817 	pddcb->acfunc = req->cmd.acfunc;	/* functional unit */
818 
819 	/*
820 	 * We know that we can get retc 0x104 with CRC error, do not
821 	 * stop the queue in those cases for this command. XDIR = 1
822 	 * does not work for old SLU versions.
823 	 *
824 	 * Last bitstream with the old XDIR behavior had SLU_ID
825 	 * 0x34199.
826 	 */
827 	if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull)
828 		pddcb->xdir = 0x1;
829 	else
830 		pddcb->xdir = 0x0;
831 
832 
833 	pddcb->psp = (((req->cmd.asiv_length / 8) << 4) |
834 		      ((req->cmd.asv_length  / 8)));
835 	pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts);
836 
837 	/*
838 	 * If copying the whole DDCB_ASIV_LENGTH is impacting
839 	 * performance we need to change it to
840 	 * req->cmd.asiv_length. But simulation benefits from some
841 	 * non-architectured bits behind the architectured content.
842 	 *
843 	 * How much data is copied depends on the availability of the
844 	 * ATS field, which was introduced late. If the ATS field is
845 	 * supported ASIV is 8 bytes shorter than it used to be. Since
846 	 * the ATS field is copied too, the code should do exactly
847 	 * what it did before, but I wanted to make copying of the ATS
848 	 * field very explicit.
849 	 */
850 	if (genwqe_get_slu_id(cd) <= 0x2) {
851 		memcpy(&pddcb->__asiv[0],	/* destination */
852 		       &req->cmd.__asiv[0],	/* source */
853 		       DDCB_ASIV_LENGTH);	/* req->cmd.asiv_length */
854 	} else {
855 		pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats);
856 		memcpy(&pddcb->n.asiv[0],	/* destination */
857 			&req->cmd.asiv[0],	/* source */
858 			DDCB_ASIV_LENGTH_ATS);	/* req->cmd.asiv_length */
859 	}
860 
861 	pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */
862 
863 	/*
864 	 * Calculate CRC_16 for corresponding range PSP(7:4). Include
865 	 * empty 4 bytes prior to the data.
866 	 */
867 	icrc = genwqe_crc16((const u8 *)pddcb,
868 			   ICRC_LENGTH(req->cmd.asiv_length), 0xffff);
869 	pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16);
870 
871 	/* enable DDCB completion irq */
872 	if (!GENWQE_POLLING_ENABLED)
873 		pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32;
874 
875 	dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num);
876 	genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
877 
878 	if (ddcb_requ_collect_debug_data(req)) {
879 		/* use the kernel copy of debug data. copying back to
880 		   user buffer happens later */
881 
882 		genwqe_init_debug_data(cd, &req->debug_data);
883 		memcpy(&req->debug_data.ddcb_before, pddcb,
884 		       sizeof(req->debug_data.ddcb_before));
885 	}
886 
887 	enqueue_ddcb(cd, queue, pddcb, req->num);
888 	queue->ddcbs_in_flight++;
889 
890 	if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight)
891 		queue->ddcbs_max_in_flight = queue->ddcbs_in_flight;
892 
893 	ddcb_requ_set_state(req, GENWQE_REQU_TAPPED);
894 	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
895 	wake_up_interruptible(&cd->queue_waitq);
896 
897 	return 0;
898 }
899 
900 /**
901  * __genwqe_execute_raw_ddcb() - Setup and execute DDCB
902  * @cd:         pointer to genwqe device descriptor
903  * @req:        user provided DDCB request
904  * @f_flags:    file mode: blocking, non-blocking
905  */
906 int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd,
907 			      struct genwqe_ddcb_cmd *cmd,
908 			      unsigned int f_flags)
909 {
910 	int rc = 0;
911 	struct pci_dev *pci_dev = cd->pci_dev;
912 	struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
913 
914 	if (cmd->asiv_length > DDCB_ASIV_LENGTH) {
915 		dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n",
916 			__func__, cmd->asiv_length);
917 		return -EINVAL;
918 	}
919 	if (cmd->asv_length > DDCB_ASV_LENGTH) {
920 		dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n",
921 			__func__, cmd->asiv_length);
922 		return -EINVAL;
923 	}
924 	rc = __genwqe_enqueue_ddcb(cd, req, f_flags);
925 	if (rc != 0)
926 		return rc;
927 
928 	rc = __genwqe_wait_ddcb(cd, req);
929 	if (rc < 0)		/* error or signal interrupt */
930 		goto err_exit;
931 
932 	if (ddcb_requ_collect_debug_data(req)) {
933 		if (copy_to_user((struct genwqe_debug_data __user *)
934 				 (unsigned long)cmd->ddata_addr,
935 				 &req->debug_data,
936 				 sizeof(struct genwqe_debug_data)))
937 			return -EFAULT;
938 	}
939 
940 	/*
941 	 * Higher values than 0x102 indicate completion with faults,
942 	 * lower values than 0x102 indicate processing faults. Note
943 	 * that DDCB might have been purged. E.g. Cntl+C.
944 	 */
945 	if (cmd->retc != DDCB_RETC_COMPLETE) {
946 		/* This might happen e.g. flash read, and needs to be
947 		   handled by the upper layer code. */
948 		rc = -EBADMSG;	/* not processed/error retc */
949 	}
950 
951 	return rc;
952 
953  err_exit:
954 	__genwqe_purge_ddcb(cd, req);
955 
956 	if (ddcb_requ_collect_debug_data(req)) {
957 		if (copy_to_user((struct genwqe_debug_data __user *)
958 				 (unsigned long)cmd->ddata_addr,
959 				 &req->debug_data,
960 				 sizeof(struct genwqe_debug_data)))
961 			return -EFAULT;
962 	}
963 	return rc;
964 }
965 
966 /**
967  * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished
968  *
969  * We use this as condition for our wait-queue code.
970  */
971 static int genwqe_next_ddcb_ready(struct genwqe_dev *cd)
972 {
973 	unsigned long flags;
974 	struct ddcb *pddcb;
975 	struct ddcb_queue *queue = &cd->queue;
976 
977 	spin_lock_irqsave(&queue->ddcb_lock, flags);
978 
979 	if (queue_empty(queue)) { /* emtpy queue */
980 		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
981 		return 0;
982 	}
983 
984 	pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
985 	if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */
986 		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
987 		return 1;
988 	}
989 
990 	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
991 	return 0;
992 }
993 
994 /**
995  * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight
996  *
997  * Keep track on the number of DDCBs which ware currently in the
998  * queue. This is needed for statistics as well as conditon if we want
999  * to wait or better do polling in case of no interrupts available.
1000  */
1001 int genwqe_ddcbs_in_flight(struct genwqe_dev *cd)
1002 {
1003 	unsigned long flags;
1004 	int ddcbs_in_flight = 0;
1005 	struct ddcb_queue *queue = &cd->queue;
1006 
1007 	spin_lock_irqsave(&queue->ddcb_lock, flags);
1008 	ddcbs_in_flight += queue->ddcbs_in_flight;
1009 	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1010 
1011 	return ddcbs_in_flight;
1012 }
1013 
1014 static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1015 {
1016 	int rc, i;
1017 	struct ddcb *pddcb;
1018 	u64 val64;
1019 	unsigned int queue_size;
1020 	struct pci_dev *pci_dev = cd->pci_dev;
1021 
1022 	if (GENWQE_DDCB_MAX < 2)
1023 		return -EINVAL;
1024 
1025 	queue_size = roundup(GENWQE_DDCB_MAX * sizeof(struct ddcb), PAGE_SIZE);
1026 
1027 	queue->ddcbs_in_flight = 0;  /* statistics */
1028 	queue->ddcbs_max_in_flight = 0;
1029 	queue->ddcbs_completed = 0;
1030 	queue->return_on_busy = 0;
1031 	queue->wait_on_busy = 0;
1032 
1033 	queue->ddcb_seq	  = 0x100; /* start sequence number */
1034 	queue->ddcb_max	  = GENWQE_DDCB_MAX;
1035 	queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size,
1036 						&queue->ddcb_daddr);
1037 	if (queue->ddcb_vaddr == NULL) {
1038 		dev_err(&pci_dev->dev,
1039 			"[%s] **err: could not allocate DDCB **\n", __func__);
1040 		return -ENOMEM;
1041 	}
1042 	queue->ddcb_req = kcalloc(queue->ddcb_max, sizeof(struct ddcb_requ *),
1043 				  GFP_KERNEL);
1044 	if (!queue->ddcb_req) {
1045 		rc = -ENOMEM;
1046 		goto free_ddcbs;
1047 	}
1048 
1049 	queue->ddcb_waitqs = kcalloc(queue->ddcb_max,
1050 				     sizeof(wait_queue_head_t),
1051 				     GFP_KERNEL);
1052 	if (!queue->ddcb_waitqs) {
1053 		rc = -ENOMEM;
1054 		goto free_requs;
1055 	}
1056 
1057 	for (i = 0; i < queue->ddcb_max; i++) {
1058 		pddcb = &queue->ddcb_vaddr[i];		     /* DDCBs */
1059 		pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32;
1060 		pddcb->retc_16 = cpu_to_be16(0xfff);
1061 
1062 		queue->ddcb_req[i] = NULL;		     /* requests */
1063 		init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */
1064 	}
1065 
1066 	queue->ddcb_act  = 0;
1067 	queue->ddcb_next = 0;	/* queue is empty */
1068 
1069 	spin_lock_init(&queue->ddcb_lock);
1070 	init_waitqueue_head(&queue->busy_waitq);
1071 
1072 	val64 = ((u64)(queue->ddcb_max - 1) <<  8); /* lastptr */
1073 	__genwqe_writeq(cd, queue->IO_QUEUE_CONFIG,  0x07);  /* iCRC/vCRC */
1074 	__genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr);
1075 	__genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq);
1076 	__genwqe_writeq(cd, queue->IO_QUEUE_WRAP,    val64);
1077 	return 0;
1078 
1079  free_requs:
1080 	kfree(queue->ddcb_req);
1081 	queue->ddcb_req = NULL;
1082  free_ddcbs:
1083 	__genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1084 				queue->ddcb_daddr);
1085 	queue->ddcb_vaddr = NULL;
1086 	queue->ddcb_daddr = 0ull;
1087 	return rc;
1088 
1089 }
1090 
1091 static int ddcb_queue_initialized(struct ddcb_queue *queue)
1092 {
1093 	return queue->ddcb_vaddr != NULL;
1094 }
1095 
1096 static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1097 {
1098 	unsigned int queue_size;
1099 
1100 	queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE);
1101 
1102 	kfree(queue->ddcb_req);
1103 	queue->ddcb_req = NULL;
1104 
1105 	if (queue->ddcb_vaddr) {
1106 		__genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1107 					queue->ddcb_daddr);
1108 		queue->ddcb_vaddr = NULL;
1109 		queue->ddcb_daddr = 0ull;
1110 	}
1111 }
1112 
1113 static irqreturn_t genwqe_pf_isr(int irq, void *dev_id)
1114 {
1115 	u64 gfir;
1116 	struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1117 	struct pci_dev *pci_dev = cd->pci_dev;
1118 
1119 	/*
1120 	 * In case of fatal FIR error the queue is stopped, such that
1121 	 * we can safely check it without risking anything.
1122 	 */
1123 	cd->irqs_processed++;
1124 	wake_up_interruptible(&cd->queue_waitq);
1125 
1126 	/*
1127 	 * Checking for errors before kicking the queue might be
1128 	 * safer, but slower for the good-case ... See above.
1129 	 */
1130 	gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
1131 	if (((gfir & GFIR_ERR_TRIGGER) != 0x0) &&
1132 	    !pci_channel_offline(pci_dev)) {
1133 
1134 		if (cd->use_platform_recovery) {
1135 			/*
1136 			 * Since we use raw accessors, EEH errors won't be
1137 			 * detected by the platform until we do a non-raw
1138 			 * MMIO or config space read
1139 			 */
1140 			readq(cd->mmio + IO_SLC_CFGREG_GFIR);
1141 
1142 			/* Don't do anything if the PCI channel is frozen */
1143 			if (pci_channel_offline(pci_dev))
1144 				goto exit;
1145 		}
1146 
1147 		wake_up_interruptible(&cd->health_waitq);
1148 
1149 		/*
1150 		 * By default GFIRs causes recovery actions. This
1151 		 * count is just for debug when recovery is masked.
1152 		 */
1153 		dev_err_ratelimited(&pci_dev->dev,
1154 				    "[%s] GFIR=%016llx\n",
1155 				    __func__, gfir);
1156 	}
1157 
1158  exit:
1159 	return IRQ_HANDLED;
1160 }
1161 
1162 static irqreturn_t genwqe_vf_isr(int irq, void *dev_id)
1163 {
1164 	struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1165 
1166 	cd->irqs_processed++;
1167 	wake_up_interruptible(&cd->queue_waitq);
1168 
1169 	return IRQ_HANDLED;
1170 }
1171 
1172 /**
1173  * genwqe_card_thread() - Work thread for the DDCB queue
1174  *
1175  * The idea is to check if there are DDCBs in processing. If there are
1176  * some finished DDCBs, we process them and wakeup the
1177  * requestors. Otherwise we give other processes time using
1178  * cond_resched().
1179  */
1180 static int genwqe_card_thread(void *data)
1181 {
1182 	int should_stop = 0;
1183 	struct genwqe_dev *cd = (struct genwqe_dev *)data;
1184 
1185 	while (!kthread_should_stop()) {
1186 
1187 		genwqe_check_ddcb_queue(cd, &cd->queue);
1188 
1189 		if (GENWQE_POLLING_ENABLED) {
1190 			wait_event_interruptible_timeout(
1191 				cd->queue_waitq,
1192 				genwqe_ddcbs_in_flight(cd) ||
1193 				(should_stop = kthread_should_stop()), 1);
1194 		} else {
1195 			wait_event_interruptible_timeout(
1196 				cd->queue_waitq,
1197 				genwqe_next_ddcb_ready(cd) ||
1198 				(should_stop = kthread_should_stop()), HZ);
1199 		}
1200 		if (should_stop)
1201 			break;
1202 
1203 		/*
1204 		 * Avoid soft lockups on heavy loads; we do not want
1205 		 * to disable our interrupts.
1206 		 */
1207 		cond_resched();
1208 	}
1209 	return 0;
1210 }
1211 
1212 /**
1213  * genwqe_setup_service_layer() - Setup DDCB queue
1214  * @cd:         pointer to genwqe device descriptor
1215  *
1216  * Allocate DDCBs. Configure Service Layer Controller (SLC).
1217  *
1218  * Return: 0 success
1219  */
1220 int genwqe_setup_service_layer(struct genwqe_dev *cd)
1221 {
1222 	int rc;
1223 	struct ddcb_queue *queue;
1224 	struct pci_dev *pci_dev = cd->pci_dev;
1225 
1226 	if (genwqe_is_privileged(cd)) {
1227 		rc = genwqe_card_reset(cd);
1228 		if (rc < 0) {
1229 			dev_err(&pci_dev->dev,
1230 				"[%s] err: reset failed.\n", __func__);
1231 			return rc;
1232 		}
1233 		genwqe_read_softreset(cd);
1234 	}
1235 
1236 	queue = &cd->queue;
1237 	queue->IO_QUEUE_CONFIG  = IO_SLC_QUEUE_CONFIG;
1238 	queue->IO_QUEUE_STATUS  = IO_SLC_QUEUE_STATUS;
1239 	queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT;
1240 	queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN;
1241 	queue->IO_QUEUE_OFFSET  = IO_SLC_QUEUE_OFFSET;
1242 	queue->IO_QUEUE_WRAP    = IO_SLC_QUEUE_WRAP;
1243 	queue->IO_QUEUE_WTIME   = IO_SLC_QUEUE_WTIME;
1244 	queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS;
1245 	queue->IO_QUEUE_LRW     = IO_SLC_QUEUE_LRW;
1246 
1247 	rc = setup_ddcb_queue(cd, queue);
1248 	if (rc != 0) {
1249 		rc = -ENODEV;
1250 		goto err_out;
1251 	}
1252 
1253 	init_waitqueue_head(&cd->queue_waitq);
1254 	cd->card_thread = kthread_run(genwqe_card_thread, cd,
1255 				      GENWQE_DEVNAME "%d_thread",
1256 				      cd->card_idx);
1257 	if (IS_ERR(cd->card_thread)) {
1258 		rc = PTR_ERR(cd->card_thread);
1259 		cd->card_thread = NULL;
1260 		goto stop_free_queue;
1261 	}
1262 
1263 	rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS);
1264 	if (rc)
1265 		goto stop_kthread;
1266 
1267 	/*
1268 	 * We must have all wait-queues initialized when we enable the
1269 	 * interrupts. Otherwise we might crash if we get an early
1270 	 * irq.
1271 	 */
1272 	init_waitqueue_head(&cd->health_waitq);
1273 
1274 	if (genwqe_is_privileged(cd)) {
1275 		rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED,
1276 				 GENWQE_DEVNAME, cd);
1277 	} else {
1278 		rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED,
1279 				 GENWQE_DEVNAME, cd);
1280 	}
1281 	if (rc < 0) {
1282 		dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq);
1283 		goto stop_irq_cap;
1284 	}
1285 
1286 	cd->card_state = GENWQE_CARD_USED;
1287 	return 0;
1288 
1289  stop_irq_cap:
1290 	genwqe_reset_interrupt_capability(cd);
1291  stop_kthread:
1292 	kthread_stop(cd->card_thread);
1293 	cd->card_thread = NULL;
1294  stop_free_queue:
1295 	free_ddcb_queue(cd, queue);
1296  err_out:
1297 	return rc;
1298 }
1299 
1300 /**
1301  * queue_wake_up_all() - Handles fatal error case
1302  *
1303  * The PCI device got unusable and we have to stop all pending
1304  * requests as fast as we can. The code after this must purge the
1305  * DDCBs in question and ensure that all mappings are freed.
1306  */
1307 static int queue_wake_up_all(struct genwqe_dev *cd)
1308 {
1309 	unsigned int i;
1310 	unsigned long flags;
1311 	struct ddcb_queue *queue = &cd->queue;
1312 
1313 	spin_lock_irqsave(&queue->ddcb_lock, flags);
1314 
1315 	for (i = 0; i < queue->ddcb_max; i++)
1316 		wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
1317 
1318 	wake_up_interruptible(&queue->busy_waitq);
1319 	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1320 
1321 	return 0;
1322 }
1323 
1324 /**
1325  * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces
1326  *
1327  * Relies on the pre-condition that there are no users of the card
1328  * device anymore e.g. with open file-descriptors.
1329  *
1330  * This function must be robust enough to be called twice.
1331  */
1332 int genwqe_finish_queue(struct genwqe_dev *cd)
1333 {
1334 	int i, rc = 0, in_flight;
1335 	int waitmax = GENWQE_DDCB_SOFTWARE_TIMEOUT;
1336 	struct pci_dev *pci_dev = cd->pci_dev;
1337 	struct ddcb_queue *queue = &cd->queue;
1338 
1339 	if (!ddcb_queue_initialized(queue))
1340 		return 0;
1341 
1342 	/* Do not wipe out the error state. */
1343 	if (cd->card_state == GENWQE_CARD_USED)
1344 		cd->card_state = GENWQE_CARD_UNUSED;
1345 
1346 	/* Wake up all requests in the DDCB queue such that they
1347 	   should be removed nicely. */
1348 	queue_wake_up_all(cd);
1349 
1350 	/* We must wait to get rid of the DDCBs in flight */
1351 	for (i = 0; i < waitmax; i++) {
1352 		in_flight = genwqe_ddcbs_in_flight(cd);
1353 
1354 		if (in_flight == 0)
1355 			break;
1356 
1357 		dev_dbg(&pci_dev->dev,
1358 			"  DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n",
1359 			i, waitmax, in_flight);
1360 
1361 		/*
1362 		 * Severe severe error situation: The card itself has
1363 		 * 16 DDCB queues, each queue has e.g. 32 entries,
1364 		 * each DDBC has a hardware timeout of currently 250
1365 		 * msec but the PFs have a hardware timeout of 8 sec
1366 		 * ... so I take something large.
1367 		 */
1368 		msleep(1000);
1369 	}
1370 	if (i == waitmax) {
1371 		dev_err(&pci_dev->dev, "  [%s] err: queue is not empty!!\n",
1372 			__func__);
1373 		rc = -EIO;
1374 	}
1375 	return rc;
1376 }
1377 
1378 /**
1379  * genwqe_release_service_layer() - Shutdown DDCB queue
1380  * @cd:       genwqe device descriptor
1381  *
1382  * This function must be robust enough to be called twice.
1383  */
1384 int genwqe_release_service_layer(struct genwqe_dev *cd)
1385 {
1386 	struct pci_dev *pci_dev = cd->pci_dev;
1387 
1388 	if (!ddcb_queue_initialized(&cd->queue))
1389 		return 1;
1390 
1391 	free_irq(pci_dev->irq, cd);
1392 	genwqe_reset_interrupt_capability(cd);
1393 
1394 	if (cd->card_thread != NULL) {
1395 		kthread_stop(cd->card_thread);
1396 		cd->card_thread = NULL;
1397 	}
1398 
1399 	free_ddcb_queue(cd, &cd->queue);
1400 	return 0;
1401 }
1402