xref: /openbmc/linux/drivers/misc/genwqe/card_base.c (revision fa0dadde)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * IBM Accelerator Family 'GenWQE'
4  *
5  * (C) Copyright IBM Corp. 2013
6  *
7  * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
8  * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
9  * Author: Michael Jung <mijung@gmx.net>
10  * Author: Michael Ruettger <michael@ibmra.de>
11  */
12 
13 /*
14  * Module initialization and PCIe setup. Card health monitoring and
15  * recovery functionality. Character device creation and deletion are
16  * controlled from here.
17  */
18 
19 #include <linux/types.h>
20 #include <linux/pci.h>
21 #include <linux/err.h>
22 #include <linux/string.h>
23 #include <linux/sched.h>
24 #include <linux/wait.h>
25 #include <linux/delay.h>
26 #include <linux/dma-mapping.h>
27 #include <linux/module.h>
28 #include <linux/notifier.h>
29 #include <linux/device.h>
30 #include <linux/log2.h>
31 
32 #include "card_base.h"
33 #include "card_ddcb.h"
34 
35 MODULE_AUTHOR("Frank Haverkamp <haver@linux.vnet.ibm.com>");
36 MODULE_AUTHOR("Michael Ruettger <michael@ibmra.de>");
37 MODULE_AUTHOR("Joerg-Stephan Vogt <jsvogt@de.ibm.com>");
38 MODULE_AUTHOR("Michael Jung <mijung@gmx.net>");
39 
40 MODULE_DESCRIPTION("GenWQE Card");
41 MODULE_VERSION(DRV_VERSION);
42 MODULE_LICENSE("GPL");
43 
44 static char genwqe_driver_name[] = GENWQE_DEVNAME;
45 static struct class *class_genwqe;
46 static struct dentry *debugfs_genwqe;
47 static struct genwqe_dev *genwqe_devices[GENWQE_CARD_NO_MAX];
48 
49 /* PCI structure for identifying device by PCI vendor and device ID */
50 static const struct pci_device_id genwqe_device_table[] = {
51 	{ .vendor      = PCI_VENDOR_ID_IBM,
52 	  .device      = PCI_DEVICE_GENWQE,
53 	  .subvendor   = PCI_SUBVENDOR_ID_IBM,
54 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5,
55 	  .class       = (PCI_CLASSCODE_GENWQE5 << 8),
56 	  .class_mask  = ~0,
57 	  .driver_data = 0 },
58 
59 	/* Initial SR-IOV bring-up image */
60 	{ .vendor      = PCI_VENDOR_ID_IBM,
61 	  .device      = PCI_DEVICE_GENWQE,
62 	  .subvendor   = PCI_SUBVENDOR_ID_IBM_SRIOV,
63 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5_SRIOV,
64 	  .class       = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),
65 	  .class_mask  = ~0,
66 	  .driver_data = 0 },
67 
68 	{ .vendor      = PCI_VENDOR_ID_IBM,  /* VF Vendor ID */
69 	  .device      = 0x0000,  /* VF Device ID */
70 	  .subvendor   = PCI_SUBVENDOR_ID_IBM_SRIOV,
71 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5_SRIOV,
72 	  .class       = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),
73 	  .class_mask  = ~0,
74 	  .driver_data = 0 },
75 
76 	/* Fixed up image */
77 	{ .vendor      = PCI_VENDOR_ID_IBM,
78 	  .device      = PCI_DEVICE_GENWQE,
79 	  .subvendor   = PCI_SUBVENDOR_ID_IBM_SRIOV,
80 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5,
81 	  .class       = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),
82 	  .class_mask  = ~0,
83 	  .driver_data = 0 },
84 
85 	{ .vendor      = PCI_VENDOR_ID_IBM,  /* VF Vendor ID */
86 	  .device      = 0x0000,  /* VF Device ID */
87 	  .subvendor   = PCI_SUBVENDOR_ID_IBM_SRIOV,
88 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5,
89 	  .class       = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),
90 	  .class_mask  = ~0,
91 	  .driver_data = 0 },
92 
93 	/* Even one more ... */
94 	{ .vendor      = PCI_VENDOR_ID_IBM,
95 	  .device      = PCI_DEVICE_GENWQE,
96 	  .subvendor   = PCI_SUBVENDOR_ID_IBM,
97 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5_NEW,
98 	  .class       = (PCI_CLASSCODE_GENWQE5 << 8),
99 	  .class_mask  = ~0,
100 	  .driver_data = 0 },
101 
102 	{ 0, }			/* 0 terminated list. */
103 };
104 
105 MODULE_DEVICE_TABLE(pci, genwqe_device_table);
106 
107 /**
108  * genwqe_dev_alloc() - Create and prepare a new card descriptor
109  *
110  * Return: Pointer to card descriptor, or ERR_PTR(err) on error
111  */
112 static struct genwqe_dev *genwqe_dev_alloc(void)
113 {
114 	unsigned int i = 0, j;
115 	struct genwqe_dev *cd;
116 
117 	for (i = 0; i < GENWQE_CARD_NO_MAX; i++) {
118 		if (genwqe_devices[i] == NULL)
119 			break;
120 	}
121 	if (i >= GENWQE_CARD_NO_MAX)
122 		return ERR_PTR(-ENODEV);
123 
124 	cd = kzalloc(sizeof(struct genwqe_dev), GFP_KERNEL);
125 	if (!cd)
126 		return ERR_PTR(-ENOMEM);
127 
128 	cd->card_idx = i;
129 	cd->class_genwqe = class_genwqe;
130 	cd->debugfs_genwqe = debugfs_genwqe;
131 
132 	/*
133 	 * This comes from kernel config option and can be overritten via
134 	 * debugfs.
135 	 */
136 	cd->use_platform_recovery = CONFIG_GENWQE_PLATFORM_ERROR_RECOVERY;
137 
138 	init_waitqueue_head(&cd->queue_waitq);
139 
140 	spin_lock_init(&cd->file_lock);
141 	INIT_LIST_HEAD(&cd->file_list);
142 
143 	cd->card_state = GENWQE_CARD_UNUSED;
144 	spin_lock_init(&cd->print_lock);
145 
146 	cd->ddcb_software_timeout = GENWQE_DDCB_SOFTWARE_TIMEOUT;
147 	cd->kill_timeout = GENWQE_KILL_TIMEOUT;
148 
149 	for (j = 0; j < GENWQE_MAX_VFS; j++)
150 		cd->vf_jobtimeout_msec[j] = GENWQE_VF_JOBTIMEOUT_MSEC;
151 
152 	genwqe_devices[i] = cd;
153 	return cd;
154 }
155 
156 static void genwqe_dev_free(struct genwqe_dev *cd)
157 {
158 	if (!cd)
159 		return;
160 
161 	genwqe_devices[cd->card_idx] = NULL;
162 	kfree(cd);
163 }
164 
165 /**
166  * genwqe_bus_reset() - Card recovery
167  * @cd: GenWQE device information
168  *
169  * pci_reset_function() will recover the device and ensure that the
170  * registers are accessible again when it completes with success. If
171  * not, the card will stay dead and registers will be unaccessible
172  * still.
173  */
174 static int genwqe_bus_reset(struct genwqe_dev *cd)
175 {
176 	int rc = 0;
177 	struct pci_dev *pci_dev = cd->pci_dev;
178 	void __iomem *mmio;
179 
180 	if (cd->err_inject & GENWQE_INJECT_BUS_RESET_FAILURE)
181 		return -EIO;
182 
183 	mmio = cd->mmio;
184 	cd->mmio = NULL;
185 	pci_iounmap(pci_dev, mmio);
186 
187 	pci_release_mem_regions(pci_dev);
188 
189 	/*
190 	 * Firmware/BIOS might change memory mapping during bus reset.
191 	 * Settings like enable bus-mastering, ... are backuped and
192 	 * restored by the pci_reset_function().
193 	 */
194 	dev_dbg(&pci_dev->dev, "[%s] pci_reset function ...\n", __func__);
195 	rc = pci_reset_function(pci_dev);
196 	if (rc) {
197 		dev_err(&pci_dev->dev,
198 			"[%s] err: failed reset func (rc %d)\n", __func__, rc);
199 		return rc;
200 	}
201 	dev_dbg(&pci_dev->dev, "[%s] done with rc=%d\n", __func__, rc);
202 
203 	/*
204 	 * Here is the right spot to clear the register read
205 	 * failure. pci_bus_reset() does this job in real systems.
206 	 */
207 	cd->err_inject &= ~(GENWQE_INJECT_HARDWARE_FAILURE |
208 			    GENWQE_INJECT_GFIR_FATAL |
209 			    GENWQE_INJECT_GFIR_INFO);
210 
211 	rc = pci_request_mem_regions(pci_dev, genwqe_driver_name);
212 	if (rc) {
213 		dev_err(&pci_dev->dev,
214 			"[%s] err: request bars failed (%d)\n", __func__, rc);
215 		return -EIO;
216 	}
217 
218 	cd->mmio = pci_iomap(pci_dev, 0, 0);
219 	if (cd->mmio == NULL) {
220 		dev_err(&pci_dev->dev,
221 			"[%s] err: mapping BAR0 failed\n", __func__);
222 		return -ENOMEM;
223 	}
224 	return 0;
225 }
226 
227 /*
228  * Hardware circumvention section. Certain bitstreams in our test-lab
229  * had different kinds of problems. Here is where we adjust those
230  * bitstreams to function will with this version of our device driver.
231  *
232  * Thise circumventions are applied to the physical function only.
233  * The magical numbers below are identifying development/manufacturing
234  * versions of the bitstream used on the card.
235  *
236  * Turn off error reporting for old/manufacturing images.
237  */
238 
239 bool genwqe_need_err_masking(struct genwqe_dev *cd)
240 {
241 	return (cd->slu_unitcfg & 0xFFFF0ull) < 0x32170ull;
242 }
243 
244 static void genwqe_tweak_hardware(struct genwqe_dev *cd)
245 {
246 	struct pci_dev *pci_dev = cd->pci_dev;
247 
248 	/* Mask FIRs for development images */
249 	if (((cd->slu_unitcfg & 0xFFFF0ull) >= 0x32000ull) &&
250 	    ((cd->slu_unitcfg & 0xFFFF0ull) <= 0x33250ull)) {
251 		dev_warn(&pci_dev->dev,
252 			 "FIRs masked due to bitstream %016llx.%016llx\n",
253 			 cd->slu_unitcfg, cd->app_unitcfg);
254 
255 		__genwqe_writeq(cd, IO_APP_SEC_LEM_DEBUG_OVR,
256 				0xFFFFFFFFFFFFFFFFull);
257 
258 		__genwqe_writeq(cd, IO_APP_ERR_ACT_MASK,
259 				0x0000000000000000ull);
260 	}
261 }
262 
263 /**
264  * genwqe_recovery_on_fatal_gfir_required() - Version depended actions
265  * @cd: GenWQE device information
266  *
267  * Bitstreams older than 2013-02-17 have a bug where fatal GFIRs must
268  * be ignored. This is e.g. true for the bitstream we gave to the card
269  * manufacturer, but also for some old bitstreams we released to our
270  * test-lab.
271  */
272 int genwqe_recovery_on_fatal_gfir_required(struct genwqe_dev *cd)
273 {
274 	return (cd->slu_unitcfg & 0xFFFF0ull) >= 0x32170ull;
275 }
276 
277 int genwqe_flash_readback_fails(struct genwqe_dev *cd)
278 {
279 	return (cd->slu_unitcfg & 0xFFFF0ull) < 0x32170ull;
280 }
281 
282 /**
283  * genwqe_T_psec() - Calculate PF/VF timeout register content
284  * @cd: GenWQE device information
285  *
286  * Note: From a design perspective it turned out to be a bad idea to
287  * use codes here to specifiy the frequency/speed values. An old
288  * driver cannot understand new codes and is therefore always a
289  * problem. Better is to measure out the value or put the
290  * speed/frequency directly into a register which is always a valid
291  * value for old as well as for new software.
292  */
293 /* T = 1/f */
294 static int genwqe_T_psec(struct genwqe_dev *cd)
295 {
296 	u16 speed;	/* 1/f -> 250,  200,  166,  175 */
297 	static const int T[] = { 4000, 5000, 6000, 5714 };
298 
299 	speed = (u16)((cd->slu_unitcfg >> 28) & 0x0full);
300 	if (speed >= ARRAY_SIZE(T))
301 		return -1;	/* illegal value */
302 
303 	return T[speed];
304 }
305 
306 /**
307  * genwqe_setup_pf_jtimer() - Setup PF hardware timeouts for DDCB execution
308  * @cd: GenWQE device information
309  *
310  * Do this _after_ card_reset() is called. Otherwise the values will
311  * vanish. The settings need to be done when the queues are inactive.
312  *
313  * The max. timeout value is 2^(10+x) * T (6ns for 166MHz) * 15/16.
314  * The min. timeout value is 2^(10+x) * T (6ns for 166MHz) * 14/16.
315  */
316 static bool genwqe_setup_pf_jtimer(struct genwqe_dev *cd)
317 {
318 	u32 T = genwqe_T_psec(cd);
319 	u64 x;
320 
321 	if (GENWQE_PF_JOBTIMEOUT_MSEC == 0)
322 		return false;
323 
324 	/* PF: large value needed, flash update 2sec per block */
325 	x = ilog2(GENWQE_PF_JOBTIMEOUT_MSEC *
326 		  16000000000uL/(T * 15)) - 10;
327 
328 	genwqe_write_vreg(cd, IO_SLC_VF_APPJOB_TIMEOUT,
329 			  0xff00 | (x & 0xff), 0);
330 	return true;
331 }
332 
333 /**
334  * genwqe_setup_vf_jtimer() - Setup VF hardware timeouts for DDCB execution
335  * @cd: GenWQE device information
336  */
337 static bool genwqe_setup_vf_jtimer(struct genwqe_dev *cd)
338 {
339 	struct pci_dev *pci_dev = cd->pci_dev;
340 	unsigned int vf;
341 	u32 T = genwqe_T_psec(cd);
342 	u64 x;
343 	int totalvfs;
344 
345 	totalvfs = pci_sriov_get_totalvfs(pci_dev);
346 	if (totalvfs <= 0)
347 		return false;
348 
349 	for (vf = 0; vf < totalvfs; vf++) {
350 
351 		if (cd->vf_jobtimeout_msec[vf] == 0)
352 			continue;
353 
354 		x = ilog2(cd->vf_jobtimeout_msec[vf] *
355 			  16000000000uL/(T * 15)) - 10;
356 
357 		genwqe_write_vreg(cd, IO_SLC_VF_APPJOB_TIMEOUT,
358 				  0xff00 | (x & 0xff), vf + 1);
359 	}
360 	return true;
361 }
362 
363 static int genwqe_ffdc_buffs_alloc(struct genwqe_dev *cd)
364 {
365 	unsigned int type, e = 0;
366 
367 	for (type = 0; type < GENWQE_DBG_UNITS; type++) {
368 		switch (type) {
369 		case GENWQE_DBG_UNIT0:
370 			e = genwqe_ffdc_buff_size(cd, 0);
371 			break;
372 		case GENWQE_DBG_UNIT1:
373 			e = genwqe_ffdc_buff_size(cd, 1);
374 			break;
375 		case GENWQE_DBG_UNIT2:
376 			e = genwqe_ffdc_buff_size(cd, 2);
377 			break;
378 		case GENWQE_DBG_REGS:
379 			e = GENWQE_FFDC_REGS;
380 			break;
381 		}
382 
383 		/* currently support only the debug units mentioned here */
384 		cd->ffdc[type].entries = e;
385 		cd->ffdc[type].regs =
386 			kmalloc_array(e, sizeof(struct genwqe_reg),
387 				      GFP_KERNEL);
388 		/*
389 		 * regs == NULL is ok, the using code treats this as no regs,
390 		 * Printing warning is ok in this case.
391 		 */
392 	}
393 	return 0;
394 }
395 
396 static void genwqe_ffdc_buffs_free(struct genwqe_dev *cd)
397 {
398 	unsigned int type;
399 
400 	for (type = 0; type < GENWQE_DBG_UNITS; type++) {
401 		kfree(cd->ffdc[type].regs);
402 		cd->ffdc[type].regs = NULL;
403 	}
404 }
405 
406 static int genwqe_read_ids(struct genwqe_dev *cd)
407 {
408 	int err = 0;
409 	int slu_id;
410 	struct pci_dev *pci_dev = cd->pci_dev;
411 
412 	cd->slu_unitcfg = __genwqe_readq(cd, IO_SLU_UNITCFG);
413 	if (cd->slu_unitcfg == IO_ILLEGAL_VALUE) {
414 		dev_err(&pci_dev->dev,
415 			"err: SLUID=%016llx\n", cd->slu_unitcfg);
416 		err = -EIO;
417 		goto out_err;
418 	}
419 
420 	slu_id = genwqe_get_slu_id(cd);
421 	if (slu_id < GENWQE_SLU_ARCH_REQ || slu_id == 0xff) {
422 		dev_err(&pci_dev->dev,
423 			"err: incompatible SLU Architecture %u\n", slu_id);
424 		err = -ENOENT;
425 		goto out_err;
426 	}
427 
428 	cd->app_unitcfg = __genwqe_readq(cd, IO_APP_UNITCFG);
429 	if (cd->app_unitcfg == IO_ILLEGAL_VALUE) {
430 		dev_err(&pci_dev->dev,
431 			"err: APPID=%016llx\n", cd->app_unitcfg);
432 		err = -EIO;
433 		goto out_err;
434 	}
435 	genwqe_read_app_id(cd, cd->app_name, sizeof(cd->app_name));
436 
437 	/*
438 	 * Is access to all registers possible? If we are a VF the
439 	 * answer is obvious. If we run fully virtualized, we need to
440 	 * check if we can access all registers. If we do not have
441 	 * full access we will cause an UR and some informational FIRs
442 	 * in the PF, but that should not harm.
443 	 */
444 	if (pci_dev->is_virtfn)
445 		cd->is_privileged = 0;
446 	else
447 		cd->is_privileged = (__genwqe_readq(cd, IO_SLU_BITSTREAM)
448 				     != IO_ILLEGAL_VALUE);
449 
450  out_err:
451 	return err;
452 }
453 
454 static int genwqe_start(struct genwqe_dev *cd)
455 {
456 	int err;
457 	struct pci_dev *pci_dev = cd->pci_dev;
458 
459 	err = genwqe_read_ids(cd);
460 	if (err)
461 		return err;
462 
463 	if (genwqe_is_privileged(cd)) {
464 		/* do this after the tweaks. alloc fail is acceptable */
465 		genwqe_ffdc_buffs_alloc(cd);
466 		genwqe_stop_traps(cd);
467 
468 		/* Collect registers e.g. FIRs, UNITIDs, traces ... */
469 		genwqe_read_ffdc_regs(cd, cd->ffdc[GENWQE_DBG_REGS].regs,
470 				      cd->ffdc[GENWQE_DBG_REGS].entries, 0);
471 
472 		genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT0,
473 				      cd->ffdc[GENWQE_DBG_UNIT0].regs,
474 				      cd->ffdc[GENWQE_DBG_UNIT0].entries);
475 
476 		genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT1,
477 				      cd->ffdc[GENWQE_DBG_UNIT1].regs,
478 				      cd->ffdc[GENWQE_DBG_UNIT1].entries);
479 
480 		genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT2,
481 				      cd->ffdc[GENWQE_DBG_UNIT2].regs,
482 				      cd->ffdc[GENWQE_DBG_UNIT2].entries);
483 
484 		genwqe_start_traps(cd);
485 
486 		if (cd->card_state == GENWQE_CARD_FATAL_ERROR) {
487 			dev_warn(&pci_dev->dev,
488 				 "[%s] chip reload/recovery!\n", __func__);
489 
490 			/*
491 			 * Stealth Mode: Reload chip on either hot
492 			 * reset or PERST.
493 			 */
494 			cd->softreset = 0x7Cull;
495 			__genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET,
496 				       cd->softreset);
497 
498 			err = genwqe_bus_reset(cd);
499 			if (err != 0) {
500 				dev_err(&pci_dev->dev,
501 					"[%s] err: bus reset failed!\n",
502 					__func__);
503 				goto out;
504 			}
505 
506 			/*
507 			 * Re-read the IDs because
508 			 * it could happen that the bitstream load
509 			 * failed!
510 			 */
511 			err = genwqe_read_ids(cd);
512 			if (err)
513 				goto out;
514 		}
515 	}
516 
517 	err = genwqe_setup_service_layer(cd);  /* does a reset to the card */
518 	if (err != 0) {
519 		dev_err(&pci_dev->dev,
520 			"[%s] err: could not setup servicelayer!\n", __func__);
521 		err = -ENODEV;
522 		goto out;
523 	}
524 
525 	if (genwqe_is_privileged(cd)) {	 /* code is running _after_ reset */
526 		genwqe_tweak_hardware(cd);
527 
528 		genwqe_setup_pf_jtimer(cd);
529 		genwqe_setup_vf_jtimer(cd);
530 	}
531 
532 	err = genwqe_device_create(cd);
533 	if (err < 0) {
534 		dev_err(&pci_dev->dev,
535 			"err: chdev init failed! (err=%d)\n", err);
536 		goto out_release_service_layer;
537 	}
538 	return 0;
539 
540  out_release_service_layer:
541 	genwqe_release_service_layer(cd);
542  out:
543 	if (genwqe_is_privileged(cd))
544 		genwqe_ffdc_buffs_free(cd);
545 	return -EIO;
546 }
547 
548 /**
549  * genwqe_stop() - Stop card operation
550  * @cd: GenWQE device information
551  *
552  * Recovery notes:
553  *   As long as genwqe_thread runs we might access registers during
554  *   error data capture. Same is with the genwqe_health_thread.
555  *   When genwqe_bus_reset() fails this function might called two times:
556  *   first by the genwqe_health_thread() and later by genwqe_remove() to
557  *   unbind the device. We must be able to survive that.
558  *
559  * This function must be robust enough to be called twice.
560  */
561 static int genwqe_stop(struct genwqe_dev *cd)
562 {
563 	genwqe_finish_queue(cd);	    /* no register access */
564 	genwqe_device_remove(cd);	    /* device removed, procs killed */
565 	genwqe_release_service_layer(cd);   /* here genwqe_thread is stopped */
566 
567 	if (genwqe_is_privileged(cd)) {
568 		pci_disable_sriov(cd->pci_dev);	/* access pci config space */
569 		genwqe_ffdc_buffs_free(cd);
570 	}
571 
572 	return 0;
573 }
574 
575 /**
576  * genwqe_recover_card() - Try to recover the card if it is possible
577  * @cd: GenWQE device information
578  * @fatal_err: Indicate whether to attempt soft reset
579  *
580  * If fatal_err is set no register access is possible anymore. It is
581  * likely that genwqe_start fails in that situation. Proper error
582  * handling is required in this case.
583  *
584  * genwqe_bus_reset() will cause the pci code to call genwqe_remove()
585  * and later genwqe_probe() for all virtual functions.
586  */
587 static int genwqe_recover_card(struct genwqe_dev *cd, int fatal_err)
588 {
589 	int rc;
590 	struct pci_dev *pci_dev = cd->pci_dev;
591 
592 	genwqe_stop(cd);
593 
594 	/*
595 	 * Make sure chip is not reloaded to maintain FFDC. Write SLU
596 	 * Reset Register, CPLDReset field to 0.
597 	 */
598 	if (!fatal_err) {
599 		cd->softreset = 0x70ull;
600 		__genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, cd->softreset);
601 	}
602 
603 	rc = genwqe_bus_reset(cd);
604 	if (rc != 0) {
605 		dev_err(&pci_dev->dev,
606 			"[%s] err: card recovery impossible!\n", __func__);
607 		return rc;
608 	}
609 
610 	rc = genwqe_start(cd);
611 	if (rc < 0) {
612 		dev_err(&pci_dev->dev,
613 			"[%s] err: failed to launch device!\n", __func__);
614 		return rc;
615 	}
616 	return 0;
617 }
618 
619 static int genwqe_health_check_cond(struct genwqe_dev *cd, u64 *gfir)
620 {
621 	*gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
622 	return (*gfir & GFIR_ERR_TRIGGER) &&
623 		genwqe_recovery_on_fatal_gfir_required(cd);
624 }
625 
626 /**
627  * genwqe_fir_checking() - Check the fault isolation registers of the card
628  * @cd: GenWQE device information
629  *
630  * If this code works ok, can be tried out with help of the genwqe_poke tool:
631  *   sudo ./tools/genwqe_poke 0x8 0xfefefefefef
632  *
633  * Now the relevant FIRs/sFIRs should be printed out and the driver should
634  * invoke recovery (devices are removed and readded).
635  */
636 static u64 genwqe_fir_checking(struct genwqe_dev *cd)
637 {
638 	int j, iterations = 0;
639 	u64 mask, fir, fec, uid, gfir, gfir_masked, sfir, sfec;
640 	u32 fir_addr, fir_clr_addr, fec_addr, sfir_addr, sfec_addr;
641 	struct pci_dev *pci_dev = cd->pci_dev;
642 
643  healthMonitor:
644 	iterations++;
645 	if (iterations > 16) {
646 		dev_err(&pci_dev->dev, "* exit looping after %d times\n",
647 			iterations);
648 		goto fatal_error;
649 	}
650 
651 	gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
652 	if (gfir != 0x0)
653 		dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n",
654 				    IO_SLC_CFGREG_GFIR, gfir);
655 	if (gfir == IO_ILLEGAL_VALUE)
656 		goto fatal_error;
657 
658 	/*
659 	 * Avoid printing when to GFIR bit is on prevents contignous
660 	 * printout e.g. for the following bug:
661 	 *   FIR set without a 2ndary FIR/FIR cannot be cleared
662 	 * Comment out the following if to get the prints:
663 	 */
664 	if (gfir == 0)
665 		return 0;
666 
667 	gfir_masked = gfir & GFIR_ERR_TRIGGER;  /* fatal errors */
668 
669 	for (uid = 0; uid < GENWQE_MAX_UNITS; uid++) { /* 0..2 in zEDC */
670 
671 		/* read the primary FIR (pfir) */
672 		fir_addr = (uid << 24) + 0x08;
673 		fir = __genwqe_readq(cd, fir_addr);
674 		if (fir == 0x0)
675 			continue;  /* no error in this unit */
676 
677 		dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n", fir_addr, fir);
678 		if (fir == IO_ILLEGAL_VALUE)
679 			goto fatal_error;
680 
681 		/* read primary FEC */
682 		fec_addr = (uid << 24) + 0x18;
683 		fec = __genwqe_readq(cd, fec_addr);
684 
685 		dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n", fec_addr, fec);
686 		if (fec == IO_ILLEGAL_VALUE)
687 			goto fatal_error;
688 
689 		for (j = 0, mask = 1ULL; j < 64; j++, mask <<= 1) {
690 
691 			/* secondary fir empty, skip it */
692 			if ((fir & mask) == 0x0)
693 				continue;
694 
695 			sfir_addr = (uid << 24) + 0x100 + 0x08 * j;
696 			sfir = __genwqe_readq(cd, sfir_addr);
697 
698 			if (sfir == IO_ILLEGAL_VALUE)
699 				goto fatal_error;
700 			dev_err(&pci_dev->dev,
701 				"* 0x%08x 0x%016llx\n", sfir_addr, sfir);
702 
703 			sfec_addr = (uid << 24) + 0x300 + 0x08 * j;
704 			sfec = __genwqe_readq(cd, sfec_addr);
705 
706 			if (sfec == IO_ILLEGAL_VALUE)
707 				goto fatal_error;
708 			dev_err(&pci_dev->dev,
709 				"* 0x%08x 0x%016llx\n", sfec_addr, sfec);
710 
711 			gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
712 			if (gfir == IO_ILLEGAL_VALUE)
713 				goto fatal_error;
714 
715 			/* gfir turned on during routine! get out and
716 			   start over. */
717 			if ((gfir_masked == 0x0) &&
718 			    (gfir & GFIR_ERR_TRIGGER)) {
719 				goto healthMonitor;
720 			}
721 
722 			/* do not clear if we entered with a fatal gfir */
723 			if (gfir_masked == 0x0) {
724 
725 				/* NEW clear by mask the logged bits */
726 				sfir_addr = (uid << 24) + 0x100 + 0x08 * j;
727 				__genwqe_writeq(cd, sfir_addr, sfir);
728 
729 				dev_dbg(&pci_dev->dev,
730 					"[HM] Clearing  2ndary FIR 0x%08x with 0x%016llx\n",
731 					sfir_addr, sfir);
732 
733 				/*
734 				 * note, these cannot be error-Firs
735 				 * since gfir_masked is 0 after sfir
736 				 * was read. Also, it is safe to do
737 				 * this write if sfir=0. Still need to
738 				 * clear the primary. This just means
739 				 * there is no secondary FIR.
740 				 */
741 
742 				/* clear by mask the logged bit. */
743 				fir_clr_addr = (uid << 24) + 0x10;
744 				__genwqe_writeq(cd, fir_clr_addr, mask);
745 
746 				dev_dbg(&pci_dev->dev,
747 					"[HM] Clearing primary FIR 0x%08x with 0x%016llx\n",
748 					fir_clr_addr, mask);
749 			}
750 		}
751 	}
752 	gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
753 	if (gfir == IO_ILLEGAL_VALUE)
754 		goto fatal_error;
755 
756 	if ((gfir_masked == 0x0) && (gfir & GFIR_ERR_TRIGGER)) {
757 		/*
758 		 * Check once more that it didn't go on after all the
759 		 * FIRS were cleared.
760 		 */
761 		dev_dbg(&pci_dev->dev, "ACK! Another FIR! Recursing %d!\n",
762 			iterations);
763 		goto healthMonitor;
764 	}
765 	return gfir_masked;
766 
767  fatal_error:
768 	return IO_ILLEGAL_VALUE;
769 }
770 
771 /**
772  * genwqe_pci_fundamental_reset() - trigger a PCIe fundamental reset on the slot
773  * @pci_dev:	PCI device information struct
774  *
775  * Note: pci_set_pcie_reset_state() is not implemented on all archs, so this
776  * reset method will not work in all cases.
777  *
778  * Return: 0 on success or error code from pci_set_pcie_reset_state()
779  */
780 static int genwqe_pci_fundamental_reset(struct pci_dev *pci_dev)
781 {
782 	int rc;
783 
784 	/*
785 	 * lock pci config space access from userspace,
786 	 * save state and issue PCIe fundamental reset
787 	 */
788 	pci_cfg_access_lock(pci_dev);
789 	pci_save_state(pci_dev);
790 	rc = pci_set_pcie_reset_state(pci_dev, pcie_warm_reset);
791 	if (!rc) {
792 		/* keep PCIe reset asserted for 250ms */
793 		msleep(250);
794 		pci_set_pcie_reset_state(pci_dev, pcie_deassert_reset);
795 		/* Wait for 2s to reload flash and train the link */
796 		msleep(2000);
797 	}
798 	pci_restore_state(pci_dev);
799 	pci_cfg_access_unlock(pci_dev);
800 	return rc;
801 }
802 
803 
804 static int genwqe_platform_recovery(struct genwqe_dev *cd)
805 {
806 	struct pci_dev *pci_dev = cd->pci_dev;
807 	int rc;
808 
809 	dev_info(&pci_dev->dev,
810 		 "[%s] resetting card for error recovery\n", __func__);
811 
812 	/* Clear out error injection flags */
813 	cd->err_inject &= ~(GENWQE_INJECT_HARDWARE_FAILURE |
814 			    GENWQE_INJECT_GFIR_FATAL |
815 			    GENWQE_INJECT_GFIR_INFO);
816 
817 	genwqe_stop(cd);
818 
819 	/* Try recoverying the card with fundamental reset */
820 	rc = genwqe_pci_fundamental_reset(pci_dev);
821 	if (!rc) {
822 		rc = genwqe_start(cd);
823 		if (!rc)
824 			dev_info(&pci_dev->dev,
825 				 "[%s] card recovered\n", __func__);
826 		else
827 			dev_err(&pci_dev->dev,
828 				"[%s] err: cannot start card services! (err=%d)\n",
829 				__func__, rc);
830 	} else {
831 		dev_err(&pci_dev->dev,
832 			"[%s] card reset failed\n", __func__);
833 	}
834 
835 	return rc;
836 }
837 
838 /**
839  * genwqe_reload_bistream() - reload card bitstream
840  * @cd: GenWQE device information
841  *
842  * Set the appropriate register and call fundamental reset to reaload the card
843  * bitstream.
844  *
845  * Return: 0 on success, error code otherwise
846  */
847 static int genwqe_reload_bistream(struct genwqe_dev *cd)
848 {
849 	struct pci_dev *pci_dev = cd->pci_dev;
850 	int rc;
851 
852 	dev_info(&pci_dev->dev,
853 		 "[%s] resetting card for bitstream reload\n",
854 		 __func__);
855 
856 	genwqe_stop(cd);
857 
858 	/*
859 	 * Cause a CPLD reprogram with the 'next_bitstream'
860 	 * partition on PCIe hot or fundamental reset
861 	 */
862 	__genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET,
863 			(cd->softreset & 0xcull) | 0x70ull);
864 
865 	rc = genwqe_pci_fundamental_reset(pci_dev);
866 	if (rc) {
867 		/*
868 		 * A fundamental reset failure can be caused
869 		 * by lack of support on the arch, so we just
870 		 * log the error and try to start the card
871 		 * again.
872 		 */
873 		dev_err(&pci_dev->dev,
874 			"[%s] err: failed to reset card for bitstream reload\n",
875 			__func__);
876 	}
877 
878 	rc = genwqe_start(cd);
879 	if (rc) {
880 		dev_err(&pci_dev->dev,
881 			"[%s] err: cannot start card services! (err=%d)\n",
882 			__func__, rc);
883 		return rc;
884 	}
885 	dev_info(&pci_dev->dev,
886 		 "[%s] card reloaded\n", __func__);
887 	return 0;
888 }
889 
890 
891 /**
892  * genwqe_health_thread() - Health checking thread
893  * @data: GenWQE device information
894  *
895  * This thread is only started for the PF of the card.
896  *
897  * This thread monitors the health of the card. A critical situation
898  * is when we read registers which contain -1 (IO_ILLEGAL_VALUE). In
899  * this case we need to be recovered from outside. Writing to
900  * registers will very likely not work either.
901  *
902  * This thread must only exit if kthread_should_stop() becomes true.
903  *
904  * Condition for the health-thread to trigger:
905  *   a) when a kthread_stop() request comes in or
906  *   b) a critical GFIR occured
907  *
908  * Informational GFIRs are checked and potentially printed in
909  * GENWQE_HEALTH_CHECK_INTERVAL seconds.
910  */
911 static int genwqe_health_thread(void *data)
912 {
913 	int rc, should_stop = 0;
914 	struct genwqe_dev *cd = data;
915 	struct pci_dev *pci_dev = cd->pci_dev;
916 	u64 gfir, gfir_masked, slu_unitcfg, app_unitcfg;
917 
918  health_thread_begin:
919 	while (!kthread_should_stop()) {
920 		rc = wait_event_interruptible_timeout(cd->health_waitq,
921 			 (genwqe_health_check_cond(cd, &gfir) ||
922 			  (should_stop = kthread_should_stop())),
923 				GENWQE_HEALTH_CHECK_INTERVAL * HZ);
924 
925 		if (should_stop)
926 			break;
927 
928 		if (gfir == IO_ILLEGAL_VALUE) {
929 			dev_err(&pci_dev->dev,
930 				"[%s] GFIR=%016llx\n", __func__, gfir);
931 			goto fatal_error;
932 		}
933 
934 		slu_unitcfg = __genwqe_readq(cd, IO_SLU_UNITCFG);
935 		if (slu_unitcfg == IO_ILLEGAL_VALUE) {
936 			dev_err(&pci_dev->dev,
937 				"[%s] SLU_UNITCFG=%016llx\n",
938 				__func__, slu_unitcfg);
939 			goto fatal_error;
940 		}
941 
942 		app_unitcfg = __genwqe_readq(cd, IO_APP_UNITCFG);
943 		if (app_unitcfg == IO_ILLEGAL_VALUE) {
944 			dev_err(&pci_dev->dev,
945 				"[%s] APP_UNITCFG=%016llx\n",
946 				__func__, app_unitcfg);
947 			goto fatal_error;
948 		}
949 
950 		gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
951 		if (gfir == IO_ILLEGAL_VALUE) {
952 			dev_err(&pci_dev->dev,
953 				"[%s] %s: GFIR=%016llx\n", __func__,
954 				(gfir & GFIR_ERR_TRIGGER) ? "err" : "info",
955 				gfir);
956 			goto fatal_error;
957 		}
958 
959 		gfir_masked = genwqe_fir_checking(cd);
960 		if (gfir_masked == IO_ILLEGAL_VALUE)
961 			goto fatal_error;
962 
963 		/*
964 		 * GFIR ErrorTrigger bits set => reset the card!
965 		 * Never do this for old/manufacturing images!
966 		 */
967 		if ((gfir_masked) && !cd->skip_recovery &&
968 		    genwqe_recovery_on_fatal_gfir_required(cd)) {
969 
970 			cd->card_state = GENWQE_CARD_FATAL_ERROR;
971 
972 			rc = genwqe_recover_card(cd, 0);
973 			if (rc < 0) {
974 				/* FIXME Card is unusable and needs unbind! */
975 				goto fatal_error;
976 			}
977 		}
978 
979 		if (cd->card_state == GENWQE_CARD_RELOAD_BITSTREAM) {
980 			/* Userspace requested card bitstream reload */
981 			rc = genwqe_reload_bistream(cd);
982 			if (rc)
983 				goto fatal_error;
984 		}
985 
986 		cd->last_gfir = gfir;
987 		cond_resched();
988 	}
989 
990 	return 0;
991 
992  fatal_error:
993 	if (cd->use_platform_recovery) {
994 		/*
995 		 * Since we use raw accessors, EEH errors won't be detected
996 		 * by the platform until we do a non-raw MMIO or config space
997 		 * read
998 		 */
999 		readq(cd->mmio + IO_SLC_CFGREG_GFIR);
1000 
1001 		/* We do nothing if the card is going over PCI recovery */
1002 		if (pci_channel_offline(pci_dev))
1003 			return -EIO;
1004 
1005 		/*
1006 		 * If it's supported by the platform, we try a fundamental reset
1007 		 * to recover from a fatal error. Otherwise, we continue to wait
1008 		 * for an external recovery procedure to take care of it.
1009 		 */
1010 		rc = genwqe_platform_recovery(cd);
1011 		if (!rc)
1012 			goto health_thread_begin;
1013 	}
1014 
1015 	dev_err(&pci_dev->dev,
1016 		"[%s] card unusable. Please trigger unbind!\n", __func__);
1017 
1018 	/* Bring down logical devices to inform user space via udev remove. */
1019 	cd->card_state = GENWQE_CARD_FATAL_ERROR;
1020 	genwqe_stop(cd);
1021 
1022 	/* genwqe_bus_reset failed(). Now wait for genwqe_remove(). */
1023 	while (!kthread_should_stop())
1024 		cond_resched();
1025 
1026 	return -EIO;
1027 }
1028 
1029 static int genwqe_health_check_start(struct genwqe_dev *cd)
1030 {
1031 	int rc;
1032 
1033 	if (GENWQE_HEALTH_CHECK_INTERVAL <= 0)
1034 		return 0;	/* valid for disabling the service */
1035 
1036 	/* moved before request_irq() */
1037 	/* init_waitqueue_head(&cd->health_waitq); */
1038 
1039 	cd->health_thread = kthread_run(genwqe_health_thread, cd,
1040 					GENWQE_DEVNAME "%d_health",
1041 					cd->card_idx);
1042 	if (IS_ERR(cd->health_thread)) {
1043 		rc = PTR_ERR(cd->health_thread);
1044 		cd->health_thread = NULL;
1045 		return rc;
1046 	}
1047 	return 0;
1048 }
1049 
1050 static int genwqe_health_thread_running(struct genwqe_dev *cd)
1051 {
1052 	return cd->health_thread != NULL;
1053 }
1054 
1055 static int genwqe_health_check_stop(struct genwqe_dev *cd)
1056 {
1057 	if (!genwqe_health_thread_running(cd))
1058 		return -EIO;
1059 
1060 	kthread_stop(cd->health_thread);
1061 	cd->health_thread = NULL;
1062 	return 0;
1063 }
1064 
1065 /**
1066  * genwqe_pci_setup() - Allocate PCIe related resources for our card
1067  * @cd: GenWQE device information
1068  */
1069 static int genwqe_pci_setup(struct genwqe_dev *cd)
1070 {
1071 	int err;
1072 	struct pci_dev *pci_dev = cd->pci_dev;
1073 
1074 	err = pci_enable_device_mem(pci_dev);
1075 	if (err) {
1076 		dev_err(&pci_dev->dev,
1077 			"err: failed to enable pci memory (err=%d)\n", err);
1078 		goto err_out;
1079 	}
1080 
1081 	/* Reserve PCI I/O and memory resources */
1082 	err = pci_request_mem_regions(pci_dev, genwqe_driver_name);
1083 	if (err) {
1084 		dev_err(&pci_dev->dev,
1085 			"[%s] err: request bars failed (%d)\n", __func__, err);
1086 		err = -EIO;
1087 		goto err_disable_device;
1088 	}
1089 
1090 	/* check for 64-bit DMA address supported (DAC) */
1091 	/* check for 32-bit DMA address supported (SAC) */
1092 	if (dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64)) &&
1093 	    dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(32))) {
1094 		dev_err(&pci_dev->dev,
1095 			"err: neither DMA32 nor DMA64 supported\n");
1096 		err = -EIO;
1097 		goto out_release_resources;
1098 	}
1099 
1100 	pci_set_master(pci_dev);
1101 
1102 	/* EEH recovery requires PCIe fundamental reset */
1103 	pci_dev->needs_freset = 1;
1104 
1105 	/* request complete BAR-0 space (length = 0) */
1106 	cd->mmio_len = pci_resource_len(pci_dev, 0);
1107 	cd->mmio = pci_iomap(pci_dev, 0, 0);
1108 	if (cd->mmio == NULL) {
1109 		dev_err(&pci_dev->dev,
1110 			"[%s] err: mapping BAR0 failed\n", __func__);
1111 		err = -ENOMEM;
1112 		goto out_release_resources;
1113 	}
1114 
1115 	cd->num_vfs = pci_sriov_get_totalvfs(pci_dev);
1116 	if (cd->num_vfs < 0)
1117 		cd->num_vfs = 0;
1118 
1119 	err = genwqe_read_ids(cd);
1120 	if (err)
1121 		goto out_iounmap;
1122 
1123 	return 0;
1124 
1125  out_iounmap:
1126 	pci_iounmap(pci_dev, cd->mmio);
1127  out_release_resources:
1128 	pci_release_mem_regions(pci_dev);
1129  err_disable_device:
1130 	pci_disable_device(pci_dev);
1131  err_out:
1132 	return err;
1133 }
1134 
1135 /**
1136  * genwqe_pci_remove() - Free PCIe related resources for our card
1137  * @cd: GenWQE device information
1138  */
1139 static void genwqe_pci_remove(struct genwqe_dev *cd)
1140 {
1141 	struct pci_dev *pci_dev = cd->pci_dev;
1142 
1143 	if (cd->mmio)
1144 		pci_iounmap(pci_dev, cd->mmio);
1145 
1146 	pci_release_mem_regions(pci_dev);
1147 	pci_disable_device(pci_dev);
1148 }
1149 
1150 /**
1151  * genwqe_probe() - Device initialization
1152  * @pci_dev:	PCI device information struct
1153  * @id:		PCI device ID
1154  *
1155  * Callable for multiple cards. This function is called on bind.
1156  *
1157  * Return: 0 if succeeded, < 0 when failed
1158  */
1159 static int genwqe_probe(struct pci_dev *pci_dev,
1160 			const struct pci_device_id *id)
1161 {
1162 	int err;
1163 	struct genwqe_dev *cd;
1164 
1165 	genwqe_init_crc32();
1166 
1167 	cd = genwqe_dev_alloc();
1168 	if (IS_ERR(cd)) {
1169 		dev_err(&pci_dev->dev, "err: could not alloc mem (err=%d)!\n",
1170 			(int)PTR_ERR(cd));
1171 		return PTR_ERR(cd);
1172 	}
1173 
1174 	dev_set_drvdata(&pci_dev->dev, cd);
1175 	cd->pci_dev = pci_dev;
1176 
1177 	err = genwqe_pci_setup(cd);
1178 	if (err < 0) {
1179 		dev_err(&pci_dev->dev,
1180 			"err: problems with PCI setup (err=%d)\n", err);
1181 		goto out_free_dev;
1182 	}
1183 
1184 	err = genwqe_start(cd);
1185 	if (err < 0) {
1186 		dev_err(&pci_dev->dev,
1187 			"err: cannot start card services! (err=%d)\n", err);
1188 		goto out_pci_remove;
1189 	}
1190 
1191 	if (genwqe_is_privileged(cd)) {
1192 		err = genwqe_health_check_start(cd);
1193 		if (err < 0) {
1194 			dev_err(&pci_dev->dev,
1195 				"err: cannot start health checking! (err=%d)\n",
1196 				err);
1197 			goto out_stop_services;
1198 		}
1199 	}
1200 	return 0;
1201 
1202  out_stop_services:
1203 	genwqe_stop(cd);
1204  out_pci_remove:
1205 	genwqe_pci_remove(cd);
1206  out_free_dev:
1207 	genwqe_dev_free(cd);
1208 	return err;
1209 }
1210 
1211 /**
1212  * genwqe_remove() - Called when device is removed (hot-plugable)
1213  * @pci_dev:	PCI device information struct
1214  *
1215  * Or when driver is unloaded respecitively when unbind is done.
1216  */
1217 static void genwqe_remove(struct pci_dev *pci_dev)
1218 {
1219 	struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev);
1220 
1221 	genwqe_health_check_stop(cd);
1222 
1223 	/*
1224 	 * genwqe_stop() must survive if it is called twice
1225 	 * sequentially. This happens when the health thread calls it
1226 	 * and fails on genwqe_bus_reset().
1227 	 */
1228 	genwqe_stop(cd);
1229 	genwqe_pci_remove(cd);
1230 	genwqe_dev_free(cd);
1231 }
1232 
1233 /**
1234  * genwqe_err_error_detected() - Error detection callback
1235  * @pci_dev:	PCI device information struct
1236  * @state:	PCI channel state
1237  *
1238  * This callback is called by the PCI subsystem whenever a PCI bus
1239  * error is detected.
1240  */
1241 static pci_ers_result_t genwqe_err_error_detected(struct pci_dev *pci_dev,
1242 						 pci_channel_state_t state)
1243 {
1244 	struct genwqe_dev *cd;
1245 
1246 	dev_err(&pci_dev->dev, "[%s] state=%d\n", __func__, state);
1247 
1248 	cd = dev_get_drvdata(&pci_dev->dev);
1249 	if (cd == NULL)
1250 		return PCI_ERS_RESULT_DISCONNECT;
1251 
1252 	/* Stop the card */
1253 	genwqe_health_check_stop(cd);
1254 	genwqe_stop(cd);
1255 
1256 	/*
1257 	 * On permanent failure, the PCI code will call device remove
1258 	 * after the return of this function.
1259 	 * genwqe_stop() can be called twice.
1260 	 */
1261 	if (state == pci_channel_io_perm_failure) {
1262 		return PCI_ERS_RESULT_DISCONNECT;
1263 	} else {
1264 		genwqe_pci_remove(cd);
1265 		return PCI_ERS_RESULT_NEED_RESET;
1266 	}
1267 }
1268 
1269 static pci_ers_result_t genwqe_err_slot_reset(struct pci_dev *pci_dev)
1270 {
1271 	int rc;
1272 	struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev);
1273 
1274 	rc = genwqe_pci_setup(cd);
1275 	if (!rc) {
1276 		return PCI_ERS_RESULT_RECOVERED;
1277 	} else {
1278 		dev_err(&pci_dev->dev,
1279 			"err: problems with PCI setup (err=%d)\n", rc);
1280 		return PCI_ERS_RESULT_DISCONNECT;
1281 	}
1282 }
1283 
1284 static pci_ers_result_t genwqe_err_result_none(struct pci_dev *dev)
1285 {
1286 	return PCI_ERS_RESULT_NONE;
1287 }
1288 
1289 static void genwqe_err_resume(struct pci_dev *pci_dev)
1290 {
1291 	int rc;
1292 	struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev);
1293 
1294 	rc = genwqe_start(cd);
1295 	if (!rc) {
1296 		rc = genwqe_health_check_start(cd);
1297 		if (rc)
1298 			dev_err(&pci_dev->dev,
1299 				"err: cannot start health checking! (err=%d)\n",
1300 				rc);
1301 	} else {
1302 		dev_err(&pci_dev->dev,
1303 			"err: cannot start card services! (err=%d)\n", rc);
1304 	}
1305 }
1306 
1307 static int genwqe_sriov_configure(struct pci_dev *dev, int numvfs)
1308 {
1309 	int rc;
1310 	struct genwqe_dev *cd = dev_get_drvdata(&dev->dev);
1311 
1312 	if (numvfs > 0) {
1313 		genwqe_setup_vf_jtimer(cd);
1314 		rc = pci_enable_sriov(dev, numvfs);
1315 		if (rc < 0)
1316 			return rc;
1317 		return numvfs;
1318 	}
1319 	if (numvfs == 0) {
1320 		pci_disable_sriov(dev);
1321 		return 0;
1322 	}
1323 	return 0;
1324 }
1325 
1326 static const struct pci_error_handlers genwqe_err_handler = {
1327 	.error_detected = genwqe_err_error_detected,
1328 	.mmio_enabled	= genwqe_err_result_none,
1329 	.slot_reset	= genwqe_err_slot_reset,
1330 	.resume		= genwqe_err_resume,
1331 };
1332 
1333 static struct pci_driver genwqe_driver = {
1334 	.name	  = genwqe_driver_name,
1335 	.id_table = genwqe_device_table,
1336 	.probe	  = genwqe_probe,
1337 	.remove	  = genwqe_remove,
1338 	.sriov_configure = genwqe_sriov_configure,
1339 	.err_handler = &genwqe_err_handler,
1340 };
1341 
1342 /**
1343  * genwqe_devnode() - Set default access mode for genwqe devices.
1344  * @dev:	Pointer to device (unused)
1345  * @mode:	Carrier to pass-back given mode (permissions)
1346  *
1347  * Default mode should be rw for everybody. Do not change default
1348  * device name.
1349  */
1350 static char *genwqe_devnode(const struct device *dev, umode_t *mode)
1351 {
1352 	if (mode)
1353 		*mode = 0666;
1354 	return NULL;
1355 }
1356 
1357 /**
1358  * genwqe_init_module() - Driver registration and initialization
1359  */
1360 static int __init genwqe_init_module(void)
1361 {
1362 	int rc;
1363 
1364 	class_genwqe = class_create(GENWQE_DEVNAME);
1365 	if (IS_ERR(class_genwqe)) {
1366 		pr_err("[%s] create class failed\n", __func__);
1367 		return -ENOMEM;
1368 	}
1369 
1370 	class_genwqe->devnode = genwqe_devnode;
1371 
1372 	debugfs_genwqe = debugfs_create_dir(GENWQE_DEVNAME, NULL);
1373 
1374 	rc = pci_register_driver(&genwqe_driver);
1375 	if (rc != 0) {
1376 		pr_err("[%s] pci_reg_driver (rc=%d)\n", __func__, rc);
1377 		goto err_out0;
1378 	}
1379 
1380 	return rc;
1381 
1382  err_out0:
1383 	debugfs_remove(debugfs_genwqe);
1384 	class_destroy(class_genwqe);
1385 	return rc;
1386 }
1387 
1388 /**
1389  * genwqe_exit_module() - Driver exit
1390  */
1391 static void __exit genwqe_exit_module(void)
1392 {
1393 	pci_unregister_driver(&genwqe_driver);
1394 	debugfs_remove(debugfs_genwqe);
1395 	class_destroy(class_genwqe);
1396 }
1397 
1398 module_init(genwqe_init_module);
1399 module_exit(genwqe_exit_module);
1400