1 // SPDX-License-Identifier: GPL-2.0-only 2 /** 3 * IBM Accelerator Family 'GenWQE' 4 * 5 * (C) Copyright IBM Corp. 2013 6 * 7 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com> 8 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com> 9 * Author: Michael Jung <mijung@gmx.net> 10 * Author: Michael Ruettger <michael@ibmra.de> 11 */ 12 13 /* 14 * Device Driver Control Block (DDCB) queue support. Definition of 15 * interrupt handlers for queue support as well as triggering the 16 * health monitor code in case of problems. The current hardware uses 17 * an MSI interrupt which is shared between error handling and 18 * functional code. 19 */ 20 21 #include <linux/types.h> 22 #include <linux/sched.h> 23 #include <linux/wait.h> 24 #include <linux/pci.h> 25 #include <linux/string.h> 26 #include <linux/dma-mapping.h> 27 #include <linux/delay.h> 28 #include <linux/module.h> 29 #include <linux/interrupt.h> 30 #include <linux/crc-itu-t.h> 31 32 #include "card_base.h" 33 #include "card_ddcb.h" 34 35 /* 36 * N: next DDCB, this is where the next DDCB will be put. 37 * A: active DDCB, this is where the code will look for the next completion. 38 * x: DDCB is enqueued, we are waiting for its completion. 39 40 * Situation (1): Empty queue 41 * +---+---+---+---+---+---+---+---+ 42 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 43 * | | | | | | | | | 44 * +---+---+---+---+---+---+---+---+ 45 * A/N 46 * enqueued_ddcbs = A - N = 2 - 2 = 0 47 * 48 * Situation (2): Wrapped, N > A 49 * +---+---+---+---+---+---+---+---+ 50 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 51 * | | | x | x | | | | | 52 * +---+---+---+---+---+---+---+---+ 53 * A N 54 * enqueued_ddcbs = N - A = 4 - 2 = 2 55 * 56 * Situation (3): Queue wrapped, A > N 57 * +---+---+---+---+---+---+---+---+ 58 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 59 * | x | x | | | x | x | x | x | 60 * +---+---+---+---+---+---+---+---+ 61 * N A 62 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 2) = 6 63 * 64 * Situation (4a): Queue full N > A 65 * +---+---+---+---+---+---+---+---+ 66 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 67 * | x | x | x | x | x | x | x | | 68 * +---+---+---+---+---+---+---+---+ 69 * A N 70 * 71 * enqueued_ddcbs = N - A = 7 - 0 = 7 72 * 73 * Situation (4a): Queue full A > N 74 * +---+---+---+---+---+---+---+---+ 75 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 76 * | x | x | x | | x | x | x | x | 77 * +---+---+---+---+---+---+---+---+ 78 * N A 79 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7 80 */ 81 82 static int queue_empty(struct ddcb_queue *queue) 83 { 84 return queue->ddcb_next == queue->ddcb_act; 85 } 86 87 static int queue_enqueued_ddcbs(struct ddcb_queue *queue) 88 { 89 if (queue->ddcb_next >= queue->ddcb_act) 90 return queue->ddcb_next - queue->ddcb_act; 91 92 return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next); 93 } 94 95 static int queue_free_ddcbs(struct ddcb_queue *queue) 96 { 97 int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1; 98 99 if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */ 100 return 0; 101 } 102 return free_ddcbs; 103 } 104 105 /* 106 * Use of the PRIV field in the DDCB for queue debugging: 107 * 108 * (1) Trying to get rid of a DDCB which saw a timeout: 109 * pddcb->priv[6] = 0xcc; # cleared 110 * 111 * (2) Append a DDCB via NEXT bit: 112 * pddcb->priv[7] = 0xaa; # appended 113 * 114 * (3) DDCB needed tapping: 115 * pddcb->priv[7] = 0xbb; # tapped 116 * 117 * (4) DDCB marked as correctly finished: 118 * pddcb->priv[6] = 0xff; # finished 119 */ 120 121 static inline void ddcb_mark_tapped(struct ddcb *pddcb) 122 { 123 pddcb->priv[7] = 0xbb; /* tapped */ 124 } 125 126 static inline void ddcb_mark_appended(struct ddcb *pddcb) 127 { 128 pddcb->priv[7] = 0xaa; /* appended */ 129 } 130 131 static inline void ddcb_mark_cleared(struct ddcb *pddcb) 132 { 133 pddcb->priv[6] = 0xcc; /* cleared */ 134 } 135 136 static inline void ddcb_mark_finished(struct ddcb *pddcb) 137 { 138 pddcb->priv[6] = 0xff; /* finished */ 139 } 140 141 static inline void ddcb_mark_unused(struct ddcb *pddcb) 142 { 143 pddcb->priv_64 = cpu_to_be64(0); /* not tapped */ 144 } 145 146 /** 147 * genwqe_crc16() - Generate 16-bit crc as required for DDCBs 148 * @buff: pointer to data buffer 149 * @len: length of data for calculation 150 * @init: initial crc (0xffff at start) 151 * 152 * Polynomial = x^16 + x^12 + x^5 + 1 (0x1021) 153 * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff 154 * should result in a crc16 of 0x89c3 155 * 156 * Return: crc16 checksum in big endian format ! 157 */ 158 static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init) 159 { 160 return crc_itu_t(init, buff, len); 161 } 162 163 static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue) 164 { 165 int i; 166 struct ddcb *pddcb; 167 unsigned long flags; 168 struct pci_dev *pci_dev = cd->pci_dev; 169 170 spin_lock_irqsave(&cd->print_lock, flags); 171 172 dev_info(&pci_dev->dev, 173 "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n", 174 cd->card_idx, queue->ddcb_act, queue->ddcb_next); 175 176 pddcb = queue->ddcb_vaddr; 177 for (i = 0; i < queue->ddcb_max; i++) { 178 dev_err(&pci_dev->dev, 179 " %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n", 180 i == queue->ddcb_act ? '>' : ' ', 181 i, 182 be16_to_cpu(pddcb->retc_16), 183 be16_to_cpu(pddcb->seqnum_16), 184 pddcb->hsi, 185 pddcb->shi, 186 be64_to_cpu(pddcb->priv_64), 187 pddcb->cmd); 188 pddcb++; 189 } 190 spin_unlock_irqrestore(&cd->print_lock, flags); 191 } 192 193 struct genwqe_ddcb_cmd *ddcb_requ_alloc(void) 194 { 195 struct ddcb_requ *req; 196 197 req = kzalloc(sizeof(*req), GFP_KERNEL); 198 if (!req) 199 return NULL; 200 201 return &req->cmd; 202 } 203 204 void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd) 205 { 206 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd); 207 208 kfree(req); 209 } 210 211 static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req) 212 { 213 return req->req_state; 214 } 215 216 static inline void ddcb_requ_set_state(struct ddcb_requ *req, 217 enum genwqe_requ_state new_state) 218 { 219 req->req_state = new_state; 220 } 221 222 static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req) 223 { 224 return req->cmd.ddata_addr != 0x0; 225 } 226 227 /** 228 * ddcb_requ_finished() - Returns the hardware state of the associated DDCB 229 * @cd: pointer to genwqe device descriptor 230 * @req: DDCB work request 231 * 232 * Status of ddcb_requ mirrors this hardware state, but is copied in 233 * the ddcb_requ on interrupt/polling function. The lowlevel code 234 * should check the hardware state directly, the higher level code 235 * should check the copy. 236 * 237 * This function will also return true if the state of the queue is 238 * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the 239 * shutdown case. 240 */ 241 static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req) 242 { 243 return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) || 244 (cd->card_state != GENWQE_CARD_USED); 245 } 246 247 /** 248 * enqueue_ddcb() - Enqueue a DDCB 249 * @cd: pointer to genwqe device descriptor 250 * @queue: queue this operation should be done on 251 * @ddcb_no: pointer to ddcb number being tapped 252 * 253 * Start execution of DDCB by tapping or append to queue via NEXT 254 * bit. This is done by an atomic 'compare and swap' instruction and 255 * checking SHI and HSI of the previous DDCB. 256 * 257 * This function must only be called with ddcb_lock held. 258 * 259 * Return: 1 if new DDCB is appended to previous 260 * 2 if DDCB queue is tapped via register/simulation 261 */ 262 #define RET_DDCB_APPENDED 1 263 #define RET_DDCB_TAPPED 2 264 265 static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue, 266 struct ddcb *pddcb, int ddcb_no) 267 { 268 unsigned int try; 269 int prev_no; 270 struct ddcb *prev_ddcb; 271 __be32 old, new, icrc_hsi_shi; 272 u64 num; 273 274 /* 275 * For performance checks a Dispatch Timestamp can be put into 276 * DDCB It is supposed to use the SLU's free running counter, 277 * but this requires PCIe cycles. 278 */ 279 ddcb_mark_unused(pddcb); 280 281 /* check previous DDCB if already fetched */ 282 prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1; 283 prev_ddcb = &queue->ddcb_vaddr[prev_no]; 284 285 /* 286 * It might have happened that the HSI.FETCHED bit is 287 * set. Retry in this case. Therefore I expect maximum 2 times 288 * trying. 289 */ 290 ddcb_mark_appended(pddcb); 291 for (try = 0; try < 2; try++) { 292 old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */ 293 294 /* try to append via NEXT bit if prev DDCB is not completed */ 295 if ((old & DDCB_COMPLETED_BE32) != 0x00000000) 296 break; 297 298 new = (old | DDCB_NEXT_BE32); 299 300 wmb(); /* need to ensure write ordering */ 301 icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new); 302 303 if (icrc_hsi_shi == old) 304 return RET_DDCB_APPENDED; /* appended to queue */ 305 } 306 307 /* Queue must be re-started by updating QUEUE_OFFSET */ 308 ddcb_mark_tapped(pddcb); 309 num = (u64)ddcb_no << 8; 310 311 wmb(); /* need to ensure write ordering */ 312 __genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */ 313 314 return RET_DDCB_TAPPED; 315 } 316 317 /** 318 * copy_ddcb_results() - Copy output state from real DDCB to request 319 * 320 * Copy DDCB ASV to request struct. There is no endian 321 * conversion made, since data structure in ASV is still 322 * unknown here. 323 * 324 * This is needed by: 325 * - genwqe_purge_ddcb() 326 * - genwqe_check_ddcb_queue() 327 */ 328 static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no) 329 { 330 struct ddcb_queue *queue = req->queue; 331 struct ddcb *pddcb = &queue->ddcb_vaddr[req->num]; 332 333 memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH); 334 335 /* copy status flags of the variant part */ 336 req->cmd.vcrc = be16_to_cpu(pddcb->vcrc_16); 337 req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64); 338 req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64); 339 340 req->cmd.attn = be16_to_cpu(pddcb->attn_16); 341 req->cmd.progress = be32_to_cpu(pddcb->progress_32); 342 req->cmd.retc = be16_to_cpu(pddcb->retc_16); 343 344 if (ddcb_requ_collect_debug_data(req)) { 345 int prev_no = (ddcb_no == 0) ? 346 queue->ddcb_max - 1 : ddcb_no - 1; 347 struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no]; 348 349 memcpy(&req->debug_data.ddcb_finished, pddcb, 350 sizeof(req->debug_data.ddcb_finished)); 351 memcpy(&req->debug_data.ddcb_prev, prev_pddcb, 352 sizeof(req->debug_data.ddcb_prev)); 353 } 354 } 355 356 /** 357 * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work equests. 358 * @cd: pointer to genwqe device descriptor 359 * 360 * Return: Number of DDCBs which were finished 361 */ 362 static int genwqe_check_ddcb_queue(struct genwqe_dev *cd, 363 struct ddcb_queue *queue) 364 { 365 unsigned long flags; 366 int ddcbs_finished = 0; 367 struct pci_dev *pci_dev = cd->pci_dev; 368 369 spin_lock_irqsave(&queue->ddcb_lock, flags); 370 371 /* FIXME avoid soft locking CPU */ 372 while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) { 373 374 struct ddcb *pddcb; 375 struct ddcb_requ *req; 376 u16 vcrc, vcrc_16, retc_16; 377 378 pddcb = &queue->ddcb_vaddr[queue->ddcb_act]; 379 380 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 381 0x00000000) 382 goto go_home; /* not completed, continue waiting */ 383 384 wmb(); /* Add sync to decouple prev. read operations */ 385 386 /* Note: DDCB could be purged */ 387 req = queue->ddcb_req[queue->ddcb_act]; 388 if (req == NULL) { 389 /* this occurs if DDCB is purged, not an error */ 390 /* Move active DDCB further; Nothing to do anymore. */ 391 goto pick_next_one; 392 } 393 394 /* 395 * HSI=0x44 (fetched and completed), but RETC is 396 * 0x101, or even worse 0x000. 397 * 398 * In case of seeing the queue in inconsistent state 399 * we read the errcnts and the queue status to provide 400 * a trigger for our PCIe analyzer stop capturing. 401 */ 402 retc_16 = be16_to_cpu(pddcb->retc_16); 403 if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) { 404 u64 errcnts, status; 405 u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr; 406 407 errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS); 408 status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS); 409 410 dev_err(&pci_dev->dev, 411 "[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n", 412 __func__, be16_to_cpu(pddcb->seqnum_16), 413 pddcb->hsi, retc_16, errcnts, status, 414 queue->ddcb_daddr + ddcb_offs); 415 } 416 417 copy_ddcb_results(req, queue->ddcb_act); 418 queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */ 419 420 dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num); 421 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); 422 423 ddcb_mark_finished(pddcb); 424 425 /* calculate CRC_16 to see if VCRC is correct */ 426 vcrc = genwqe_crc16(pddcb->asv, 427 VCRC_LENGTH(req->cmd.asv_length), 428 0xffff); 429 vcrc_16 = be16_to_cpu(pddcb->vcrc_16); 430 if (vcrc != vcrc_16) { 431 printk_ratelimited(KERN_ERR 432 "%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n", 433 GENWQE_DEVNAME, dev_name(&pci_dev->dev), 434 pddcb->pre, VCRC_LENGTH(req->cmd.asv_length), 435 vcrc, vcrc_16); 436 } 437 438 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED); 439 queue->ddcbs_completed++; 440 queue->ddcbs_in_flight--; 441 442 /* wake up process waiting for this DDCB, and 443 processes on the busy queue */ 444 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]); 445 wake_up_interruptible(&queue->busy_waitq); 446 447 pick_next_one: 448 queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max; 449 ddcbs_finished++; 450 } 451 452 go_home: 453 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 454 return ddcbs_finished; 455 } 456 457 /** 458 * __genwqe_wait_ddcb(): Waits until DDCB is completed 459 * @cd: pointer to genwqe device descriptor 460 * @req: pointer to requsted DDCB parameters 461 * 462 * The Service Layer will update the RETC in DDCB when processing is 463 * pending or done. 464 * 465 * Return: > 0 remaining jiffies, DDCB completed 466 * -ETIMEDOUT when timeout 467 * -ERESTARTSYS when ^C 468 * -EINVAL when unknown error condition 469 * 470 * When an error is returned the called needs to ensure that 471 * purge_ddcb() is being called to get the &req removed from the 472 * queue. 473 */ 474 int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req) 475 { 476 int rc; 477 unsigned int ddcb_no; 478 struct ddcb_queue *queue; 479 struct pci_dev *pci_dev = cd->pci_dev; 480 481 if (req == NULL) 482 return -EINVAL; 483 484 queue = req->queue; 485 if (queue == NULL) 486 return -EINVAL; 487 488 ddcb_no = req->num; 489 if (ddcb_no >= queue->ddcb_max) 490 return -EINVAL; 491 492 rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no], 493 ddcb_requ_finished(cd, req), 494 GENWQE_DDCB_SOFTWARE_TIMEOUT * HZ); 495 496 /* 497 * We need to distinguish 3 cases here: 498 * 1. rc == 0 timeout occured 499 * 2. rc == -ERESTARTSYS signal received 500 * 3. rc > 0 remaining jiffies condition is true 501 */ 502 if (rc == 0) { 503 struct ddcb_queue *queue = req->queue; 504 struct ddcb *pddcb; 505 506 /* 507 * Timeout may be caused by long task switching time. 508 * When timeout happens, check if the request has 509 * meanwhile completed. 510 */ 511 genwqe_check_ddcb_queue(cd, req->queue); 512 if (ddcb_requ_finished(cd, req)) 513 return rc; 514 515 dev_err(&pci_dev->dev, 516 "[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n", 517 __func__, req->num, rc, ddcb_requ_get_state(req), 518 req); 519 dev_err(&pci_dev->dev, 520 "[%s] IO_QUEUE_STATUS=0x%016llx\n", __func__, 521 __genwqe_readq(cd, queue->IO_QUEUE_STATUS)); 522 523 pddcb = &queue->ddcb_vaddr[req->num]; 524 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); 525 526 print_ddcb_info(cd, req->queue); 527 return -ETIMEDOUT; 528 529 } else if (rc == -ERESTARTSYS) { 530 return rc; 531 /* 532 * EINTR: Stops the application 533 * ERESTARTSYS: Restartable systemcall; called again 534 */ 535 536 } else if (rc < 0) { 537 dev_err(&pci_dev->dev, 538 "[%s] err: DDCB#%d unknown result (rc=%d) %d!\n", 539 __func__, req->num, rc, ddcb_requ_get_state(req)); 540 return -EINVAL; 541 } 542 543 /* Severe error occured. Driver is forced to stop operation */ 544 if (cd->card_state != GENWQE_CARD_USED) { 545 dev_err(&pci_dev->dev, 546 "[%s] err: DDCB#%d forced to stop (rc=%d)\n", 547 __func__, req->num, rc); 548 return -EIO; 549 } 550 return rc; 551 } 552 553 /** 554 * get_next_ddcb() - Get next available DDCB 555 * @cd: pointer to genwqe device descriptor 556 * 557 * DDCB's content is completely cleared but presets for PRE and 558 * SEQNUM. This function must only be called when ddcb_lock is held. 559 * 560 * Return: NULL if no empty DDCB available otherwise ptr to next DDCB. 561 */ 562 static struct ddcb *get_next_ddcb(struct genwqe_dev *cd, 563 struct ddcb_queue *queue, 564 int *num) 565 { 566 u64 *pu64; 567 struct ddcb *pddcb; 568 569 if (queue_free_ddcbs(queue) == 0) /* queue is full */ 570 return NULL; 571 572 /* find new ddcb */ 573 pddcb = &queue->ddcb_vaddr[queue->ddcb_next]; 574 575 /* if it is not completed, we are not allowed to use it */ 576 /* barrier(); */ 577 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000) 578 return NULL; 579 580 *num = queue->ddcb_next; /* internal DDCB number */ 581 queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max; 582 583 /* clear important DDCB fields */ 584 pu64 = (u64 *)pddcb; 585 pu64[0] = 0ULL; /* offs 0x00 (ICRC,HSI,SHI,...) */ 586 pu64[1] = 0ULL; /* offs 0x01 (ACFUNC,CMD...) */ 587 588 /* destroy previous results in ASV */ 589 pu64[0x80/8] = 0ULL; /* offs 0x80 (ASV + 0) */ 590 pu64[0x88/8] = 0ULL; /* offs 0x88 (ASV + 0x08) */ 591 pu64[0x90/8] = 0ULL; /* offs 0x90 (ASV + 0x10) */ 592 pu64[0x98/8] = 0ULL; /* offs 0x98 (ASV + 0x18) */ 593 pu64[0xd0/8] = 0ULL; /* offs 0xd0 (RETC,ATTN...) */ 594 595 pddcb->pre = DDCB_PRESET_PRE; /* 128 */ 596 pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++); 597 return pddcb; 598 } 599 600 /** 601 * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue 602 * @cd: genwqe device descriptor 603 * @req: DDCB request 604 * 605 * This will fail when the request was already FETCHED. In this case 606 * we need to wait until it is finished. Else the DDCB can be 607 * reused. This function also ensures that the request data structure 608 * is removed from ddcb_req[]. 609 * 610 * Do not forget to call this function when genwqe_wait_ddcb() fails, 611 * such that the request gets really removed from ddcb_req[]. 612 * 613 * Return: 0 success 614 */ 615 int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req) 616 { 617 struct ddcb *pddcb = NULL; 618 unsigned int t; 619 unsigned long flags; 620 struct ddcb_queue *queue = req->queue; 621 struct pci_dev *pci_dev = cd->pci_dev; 622 u64 queue_status; 623 __be32 icrc_hsi_shi = 0x0000; 624 __be32 old, new; 625 626 /* unsigned long flags; */ 627 if (GENWQE_DDCB_SOFTWARE_TIMEOUT <= 0) { 628 dev_err(&pci_dev->dev, 629 "[%s] err: software timeout is not set!\n", __func__); 630 return -EFAULT; 631 } 632 633 pddcb = &queue->ddcb_vaddr[req->num]; 634 635 for (t = 0; t < GENWQE_DDCB_SOFTWARE_TIMEOUT * 10; t++) { 636 637 spin_lock_irqsave(&queue->ddcb_lock, flags); 638 639 /* Check if req was meanwhile finished */ 640 if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) 641 goto go_home; 642 643 /* try to set PURGE bit if FETCHED/COMPLETED are not set */ 644 old = pddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */ 645 if ((old & DDCB_FETCHED_BE32) == 0x00000000) { 646 647 new = (old | DDCB_PURGE_BE32); 648 icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32, 649 old, new); 650 if (icrc_hsi_shi == old) 651 goto finish_ddcb; 652 } 653 654 /* normal finish with HSI bit */ 655 barrier(); 656 icrc_hsi_shi = pddcb->icrc_hsi_shi_32; 657 if (icrc_hsi_shi & DDCB_COMPLETED_BE32) 658 goto finish_ddcb; 659 660 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 661 662 /* 663 * Here the check_ddcb() function will most likely 664 * discover this DDCB to be finished some point in 665 * time. It will mark the req finished and free it up 666 * in the list. 667 */ 668 669 copy_ddcb_results(req, req->num); /* for the failing case */ 670 msleep(100); /* sleep for 1/10 second and try again */ 671 continue; 672 673 finish_ddcb: 674 copy_ddcb_results(req, req->num); 675 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED); 676 queue->ddcbs_in_flight--; 677 queue->ddcb_req[req->num] = NULL; /* delete from array */ 678 ddcb_mark_cleared(pddcb); 679 680 /* Move active DDCB further; Nothing to do here anymore. */ 681 682 /* 683 * We need to ensure that there is at least one free 684 * DDCB in the queue. To do that, we must update 685 * ddcb_act only if the COMPLETED bit is set for the 686 * DDCB we are working on else we treat that DDCB even 687 * if we PURGED it as occupied (hardware is supposed 688 * to set the COMPLETED bit yet!). 689 */ 690 icrc_hsi_shi = pddcb->icrc_hsi_shi_32; 691 if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) && 692 (queue->ddcb_act == req->num)) { 693 queue->ddcb_act = ((queue->ddcb_act + 1) % 694 queue->ddcb_max); 695 } 696 go_home: 697 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 698 return 0; 699 } 700 701 /* 702 * If the card is dead and the queue is forced to stop, we 703 * might see this in the queue status register. 704 */ 705 queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS); 706 707 dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num); 708 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); 709 710 dev_err(&pci_dev->dev, 711 "[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n", 712 __func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT, 713 queue_status); 714 715 print_ddcb_info(cd, req->queue); 716 717 return -EFAULT; 718 } 719 720 int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d) 721 { 722 int len; 723 struct pci_dev *pci_dev = cd->pci_dev; 724 725 if (d == NULL) { 726 dev_err(&pci_dev->dev, 727 "[%s] err: invalid memory for debug data!\n", 728 __func__); 729 return -EFAULT; 730 } 731 732 len = sizeof(d->driver_version); 733 snprintf(d->driver_version, len, "%s", DRV_VERSION); 734 d->slu_unitcfg = cd->slu_unitcfg; 735 d->app_unitcfg = cd->app_unitcfg; 736 return 0; 737 } 738 739 /** 740 * __genwqe_enqueue_ddcb() - Enqueue a DDCB 741 * @cd: pointer to genwqe device descriptor 742 * @req: pointer to DDCB execution request 743 * @f_flags: file mode: blocking, non-blocking 744 * 745 * Return: 0 if enqueuing succeeded 746 * -EIO if card is unusable/PCIe problems 747 * -EBUSY if enqueuing failed 748 */ 749 int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req, 750 unsigned int f_flags) 751 { 752 struct ddcb *pddcb; 753 unsigned long flags; 754 struct ddcb_queue *queue; 755 struct pci_dev *pci_dev = cd->pci_dev; 756 u16 icrc; 757 758 retry: 759 if (cd->card_state != GENWQE_CARD_USED) { 760 printk_ratelimited(KERN_ERR 761 "%s %s: [%s] Card is unusable/PCIe problem Req#%d\n", 762 GENWQE_DEVNAME, dev_name(&pci_dev->dev), 763 __func__, req->num); 764 return -EIO; 765 } 766 767 queue = req->queue = &cd->queue; 768 769 /* FIXME circumvention to improve performance when no irq is 770 * there. 771 */ 772 if (GENWQE_POLLING_ENABLED) 773 genwqe_check_ddcb_queue(cd, queue); 774 775 /* 776 * It must be ensured to process all DDCBs in successive 777 * order. Use a lock here in order to prevent nested DDCB 778 * enqueuing. 779 */ 780 spin_lock_irqsave(&queue->ddcb_lock, flags); 781 782 pddcb = get_next_ddcb(cd, queue, &req->num); /* get ptr and num */ 783 if (pddcb == NULL) { 784 int rc; 785 786 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 787 788 if (f_flags & O_NONBLOCK) { 789 queue->return_on_busy++; 790 return -EBUSY; 791 } 792 793 queue->wait_on_busy++; 794 rc = wait_event_interruptible(queue->busy_waitq, 795 queue_free_ddcbs(queue) != 0); 796 dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n", 797 __func__, rc); 798 if (rc == -ERESTARTSYS) 799 return rc; /* interrupted by a signal */ 800 801 goto retry; 802 } 803 804 if (queue->ddcb_req[req->num] != NULL) { 805 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 806 807 dev_err(&pci_dev->dev, 808 "[%s] picked DDCB %d with req=%p still in use!!\n", 809 __func__, req->num, req); 810 return -EFAULT; 811 } 812 ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED); 813 queue->ddcb_req[req->num] = req; 814 815 pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts); 816 pddcb->cmd = req->cmd.cmd; 817 pddcb->acfunc = req->cmd.acfunc; /* functional unit */ 818 819 /* 820 * We know that we can get retc 0x104 with CRC error, do not 821 * stop the queue in those cases for this command. XDIR = 1 822 * does not work for old SLU versions. 823 * 824 * Last bitstream with the old XDIR behavior had SLU_ID 825 * 0x34199. 826 */ 827 if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull) 828 pddcb->xdir = 0x1; 829 else 830 pddcb->xdir = 0x0; 831 832 833 pddcb->psp = (((req->cmd.asiv_length / 8) << 4) | 834 ((req->cmd.asv_length / 8))); 835 pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts); 836 837 /* 838 * If copying the whole DDCB_ASIV_LENGTH is impacting 839 * performance we need to change it to 840 * req->cmd.asiv_length. But simulation benefits from some 841 * non-architectured bits behind the architectured content. 842 * 843 * How much data is copied depends on the availability of the 844 * ATS field, which was introduced late. If the ATS field is 845 * supported ASIV is 8 bytes shorter than it used to be. Since 846 * the ATS field is copied too, the code should do exactly 847 * what it did before, but I wanted to make copying of the ATS 848 * field very explicit. 849 */ 850 if (genwqe_get_slu_id(cd) <= 0x2) { 851 memcpy(&pddcb->__asiv[0], /* destination */ 852 &req->cmd.__asiv[0], /* source */ 853 DDCB_ASIV_LENGTH); /* req->cmd.asiv_length */ 854 } else { 855 pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats); 856 memcpy(&pddcb->n.asiv[0], /* destination */ 857 &req->cmd.asiv[0], /* source */ 858 DDCB_ASIV_LENGTH_ATS); /* req->cmd.asiv_length */ 859 } 860 861 pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */ 862 863 /* 864 * Calculate CRC_16 for corresponding range PSP(7:4). Include 865 * empty 4 bytes prior to the data. 866 */ 867 icrc = genwqe_crc16((const u8 *)pddcb, 868 ICRC_LENGTH(req->cmd.asiv_length), 0xffff); 869 pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16); 870 871 /* enable DDCB completion irq */ 872 if (!GENWQE_POLLING_ENABLED) 873 pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32; 874 875 dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num); 876 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); 877 878 if (ddcb_requ_collect_debug_data(req)) { 879 /* use the kernel copy of debug data. copying back to 880 user buffer happens later */ 881 882 genwqe_init_debug_data(cd, &req->debug_data); 883 memcpy(&req->debug_data.ddcb_before, pddcb, 884 sizeof(req->debug_data.ddcb_before)); 885 } 886 887 enqueue_ddcb(cd, queue, pddcb, req->num); 888 queue->ddcbs_in_flight++; 889 890 if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight) 891 queue->ddcbs_max_in_flight = queue->ddcbs_in_flight; 892 893 ddcb_requ_set_state(req, GENWQE_REQU_TAPPED); 894 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 895 wake_up_interruptible(&cd->queue_waitq); 896 897 return 0; 898 } 899 900 /** 901 * __genwqe_execute_raw_ddcb() - Setup and execute DDCB 902 * @cd: pointer to genwqe device descriptor 903 * @req: user provided DDCB request 904 * @f_flags: file mode: blocking, non-blocking 905 */ 906 int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd, 907 struct genwqe_ddcb_cmd *cmd, 908 unsigned int f_flags) 909 { 910 int rc = 0; 911 struct pci_dev *pci_dev = cd->pci_dev; 912 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd); 913 914 if (cmd->asiv_length > DDCB_ASIV_LENGTH) { 915 dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n", 916 __func__, cmd->asiv_length); 917 return -EINVAL; 918 } 919 if (cmd->asv_length > DDCB_ASV_LENGTH) { 920 dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n", 921 __func__, cmd->asiv_length); 922 return -EINVAL; 923 } 924 rc = __genwqe_enqueue_ddcb(cd, req, f_flags); 925 if (rc != 0) 926 return rc; 927 928 rc = __genwqe_wait_ddcb(cd, req); 929 if (rc < 0) /* error or signal interrupt */ 930 goto err_exit; 931 932 if (ddcb_requ_collect_debug_data(req)) { 933 if (copy_to_user((struct genwqe_debug_data __user *) 934 (unsigned long)cmd->ddata_addr, 935 &req->debug_data, 936 sizeof(struct genwqe_debug_data))) 937 return -EFAULT; 938 } 939 940 /* 941 * Higher values than 0x102 indicate completion with faults, 942 * lower values than 0x102 indicate processing faults. Note 943 * that DDCB might have been purged. E.g. Cntl+C. 944 */ 945 if (cmd->retc != DDCB_RETC_COMPLETE) { 946 /* This might happen e.g. flash read, and needs to be 947 handled by the upper layer code. */ 948 rc = -EBADMSG; /* not processed/error retc */ 949 } 950 951 return rc; 952 953 err_exit: 954 __genwqe_purge_ddcb(cd, req); 955 956 if (ddcb_requ_collect_debug_data(req)) { 957 if (copy_to_user((struct genwqe_debug_data __user *) 958 (unsigned long)cmd->ddata_addr, 959 &req->debug_data, 960 sizeof(struct genwqe_debug_data))) 961 return -EFAULT; 962 } 963 return rc; 964 } 965 966 /** 967 * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished 968 * 969 * We use this as condition for our wait-queue code. 970 */ 971 static int genwqe_next_ddcb_ready(struct genwqe_dev *cd) 972 { 973 unsigned long flags; 974 struct ddcb *pddcb; 975 struct ddcb_queue *queue = &cd->queue; 976 977 spin_lock_irqsave(&queue->ddcb_lock, flags); 978 979 if (queue_empty(queue)) { /* emtpy queue */ 980 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 981 return 0; 982 } 983 984 pddcb = &queue->ddcb_vaddr[queue->ddcb_act]; 985 if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */ 986 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 987 return 1; 988 } 989 990 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 991 return 0; 992 } 993 994 /** 995 * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight 996 * 997 * Keep track on the number of DDCBs which ware currently in the 998 * queue. This is needed for statistics as well as conditon if we want 999 * to wait or better do polling in case of no interrupts available. 1000 */ 1001 int genwqe_ddcbs_in_flight(struct genwqe_dev *cd) 1002 { 1003 unsigned long flags; 1004 int ddcbs_in_flight = 0; 1005 struct ddcb_queue *queue = &cd->queue; 1006 1007 spin_lock_irqsave(&queue->ddcb_lock, flags); 1008 ddcbs_in_flight += queue->ddcbs_in_flight; 1009 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 1010 1011 return ddcbs_in_flight; 1012 } 1013 1014 static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue) 1015 { 1016 int rc, i; 1017 struct ddcb *pddcb; 1018 u64 val64; 1019 unsigned int queue_size; 1020 struct pci_dev *pci_dev = cd->pci_dev; 1021 1022 if (GENWQE_DDCB_MAX < 2) 1023 return -EINVAL; 1024 1025 queue_size = roundup(GENWQE_DDCB_MAX * sizeof(struct ddcb), PAGE_SIZE); 1026 1027 queue->ddcbs_in_flight = 0; /* statistics */ 1028 queue->ddcbs_max_in_flight = 0; 1029 queue->ddcbs_completed = 0; 1030 queue->return_on_busy = 0; 1031 queue->wait_on_busy = 0; 1032 1033 queue->ddcb_seq = 0x100; /* start sequence number */ 1034 queue->ddcb_max = GENWQE_DDCB_MAX; 1035 queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size, 1036 &queue->ddcb_daddr); 1037 if (queue->ddcb_vaddr == NULL) { 1038 dev_err(&pci_dev->dev, 1039 "[%s] **err: could not allocate DDCB **\n", __func__); 1040 return -ENOMEM; 1041 } 1042 queue->ddcb_req = kcalloc(queue->ddcb_max, sizeof(struct ddcb_requ *), 1043 GFP_KERNEL); 1044 if (!queue->ddcb_req) { 1045 rc = -ENOMEM; 1046 goto free_ddcbs; 1047 } 1048 1049 queue->ddcb_waitqs = kcalloc(queue->ddcb_max, 1050 sizeof(wait_queue_head_t), 1051 GFP_KERNEL); 1052 if (!queue->ddcb_waitqs) { 1053 rc = -ENOMEM; 1054 goto free_requs; 1055 } 1056 1057 for (i = 0; i < queue->ddcb_max; i++) { 1058 pddcb = &queue->ddcb_vaddr[i]; /* DDCBs */ 1059 pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32; 1060 pddcb->retc_16 = cpu_to_be16(0xfff); 1061 1062 queue->ddcb_req[i] = NULL; /* requests */ 1063 init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */ 1064 } 1065 1066 queue->ddcb_act = 0; 1067 queue->ddcb_next = 0; /* queue is empty */ 1068 1069 spin_lock_init(&queue->ddcb_lock); 1070 init_waitqueue_head(&queue->busy_waitq); 1071 1072 val64 = ((u64)(queue->ddcb_max - 1) << 8); /* lastptr */ 1073 __genwqe_writeq(cd, queue->IO_QUEUE_CONFIG, 0x07); /* iCRC/vCRC */ 1074 __genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr); 1075 __genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq); 1076 __genwqe_writeq(cd, queue->IO_QUEUE_WRAP, val64); 1077 return 0; 1078 1079 free_requs: 1080 kfree(queue->ddcb_req); 1081 queue->ddcb_req = NULL; 1082 free_ddcbs: 1083 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr, 1084 queue->ddcb_daddr); 1085 queue->ddcb_vaddr = NULL; 1086 queue->ddcb_daddr = 0ull; 1087 return rc; 1088 1089 } 1090 1091 static int ddcb_queue_initialized(struct ddcb_queue *queue) 1092 { 1093 return queue->ddcb_vaddr != NULL; 1094 } 1095 1096 static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue) 1097 { 1098 unsigned int queue_size; 1099 1100 queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE); 1101 1102 kfree(queue->ddcb_req); 1103 queue->ddcb_req = NULL; 1104 1105 if (queue->ddcb_vaddr) { 1106 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr, 1107 queue->ddcb_daddr); 1108 queue->ddcb_vaddr = NULL; 1109 queue->ddcb_daddr = 0ull; 1110 } 1111 } 1112 1113 static irqreturn_t genwqe_pf_isr(int irq, void *dev_id) 1114 { 1115 u64 gfir; 1116 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id; 1117 struct pci_dev *pci_dev = cd->pci_dev; 1118 1119 /* 1120 * In case of fatal FIR error the queue is stopped, such that 1121 * we can safely check it without risking anything. 1122 */ 1123 cd->irqs_processed++; 1124 wake_up_interruptible(&cd->queue_waitq); 1125 1126 /* 1127 * Checking for errors before kicking the queue might be 1128 * safer, but slower for the good-case ... See above. 1129 */ 1130 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR); 1131 if (((gfir & GFIR_ERR_TRIGGER) != 0x0) && 1132 !pci_channel_offline(pci_dev)) { 1133 1134 if (cd->use_platform_recovery) { 1135 /* 1136 * Since we use raw accessors, EEH errors won't be 1137 * detected by the platform until we do a non-raw 1138 * MMIO or config space read 1139 */ 1140 readq(cd->mmio + IO_SLC_CFGREG_GFIR); 1141 1142 /* Don't do anything if the PCI channel is frozen */ 1143 if (pci_channel_offline(pci_dev)) 1144 goto exit; 1145 } 1146 1147 wake_up_interruptible(&cd->health_waitq); 1148 1149 /* 1150 * By default GFIRs causes recovery actions. This 1151 * count is just for debug when recovery is masked. 1152 */ 1153 dev_err_ratelimited(&pci_dev->dev, 1154 "[%s] GFIR=%016llx\n", 1155 __func__, gfir); 1156 } 1157 1158 exit: 1159 return IRQ_HANDLED; 1160 } 1161 1162 static irqreturn_t genwqe_vf_isr(int irq, void *dev_id) 1163 { 1164 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id; 1165 1166 cd->irqs_processed++; 1167 wake_up_interruptible(&cd->queue_waitq); 1168 1169 return IRQ_HANDLED; 1170 } 1171 1172 /** 1173 * genwqe_card_thread() - Work thread for the DDCB queue 1174 * 1175 * The idea is to check if there are DDCBs in processing. If there are 1176 * some finished DDCBs, we process them and wakeup the 1177 * requestors. Otherwise we give other processes time using 1178 * cond_resched(). 1179 */ 1180 static int genwqe_card_thread(void *data) 1181 { 1182 int should_stop = 0; 1183 struct genwqe_dev *cd = (struct genwqe_dev *)data; 1184 1185 while (!kthread_should_stop()) { 1186 1187 genwqe_check_ddcb_queue(cd, &cd->queue); 1188 1189 if (GENWQE_POLLING_ENABLED) { 1190 wait_event_interruptible_timeout( 1191 cd->queue_waitq, 1192 genwqe_ddcbs_in_flight(cd) || 1193 (should_stop = kthread_should_stop()), 1); 1194 } else { 1195 wait_event_interruptible_timeout( 1196 cd->queue_waitq, 1197 genwqe_next_ddcb_ready(cd) || 1198 (should_stop = kthread_should_stop()), HZ); 1199 } 1200 if (should_stop) 1201 break; 1202 1203 /* 1204 * Avoid soft lockups on heavy loads; we do not want 1205 * to disable our interrupts. 1206 */ 1207 cond_resched(); 1208 } 1209 return 0; 1210 } 1211 1212 /** 1213 * genwqe_setup_service_layer() - Setup DDCB queue 1214 * @cd: pointer to genwqe device descriptor 1215 * 1216 * Allocate DDCBs. Configure Service Layer Controller (SLC). 1217 * 1218 * Return: 0 success 1219 */ 1220 int genwqe_setup_service_layer(struct genwqe_dev *cd) 1221 { 1222 int rc; 1223 struct ddcb_queue *queue; 1224 struct pci_dev *pci_dev = cd->pci_dev; 1225 1226 if (genwqe_is_privileged(cd)) { 1227 rc = genwqe_card_reset(cd); 1228 if (rc < 0) { 1229 dev_err(&pci_dev->dev, 1230 "[%s] err: reset failed.\n", __func__); 1231 return rc; 1232 } 1233 genwqe_read_softreset(cd); 1234 } 1235 1236 queue = &cd->queue; 1237 queue->IO_QUEUE_CONFIG = IO_SLC_QUEUE_CONFIG; 1238 queue->IO_QUEUE_STATUS = IO_SLC_QUEUE_STATUS; 1239 queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT; 1240 queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN; 1241 queue->IO_QUEUE_OFFSET = IO_SLC_QUEUE_OFFSET; 1242 queue->IO_QUEUE_WRAP = IO_SLC_QUEUE_WRAP; 1243 queue->IO_QUEUE_WTIME = IO_SLC_QUEUE_WTIME; 1244 queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS; 1245 queue->IO_QUEUE_LRW = IO_SLC_QUEUE_LRW; 1246 1247 rc = setup_ddcb_queue(cd, queue); 1248 if (rc != 0) { 1249 rc = -ENODEV; 1250 goto err_out; 1251 } 1252 1253 init_waitqueue_head(&cd->queue_waitq); 1254 cd->card_thread = kthread_run(genwqe_card_thread, cd, 1255 GENWQE_DEVNAME "%d_thread", 1256 cd->card_idx); 1257 if (IS_ERR(cd->card_thread)) { 1258 rc = PTR_ERR(cd->card_thread); 1259 cd->card_thread = NULL; 1260 goto stop_free_queue; 1261 } 1262 1263 rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS); 1264 if (rc) 1265 goto stop_kthread; 1266 1267 /* 1268 * We must have all wait-queues initialized when we enable the 1269 * interrupts. Otherwise we might crash if we get an early 1270 * irq. 1271 */ 1272 init_waitqueue_head(&cd->health_waitq); 1273 1274 if (genwqe_is_privileged(cd)) { 1275 rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED, 1276 GENWQE_DEVNAME, cd); 1277 } else { 1278 rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED, 1279 GENWQE_DEVNAME, cd); 1280 } 1281 if (rc < 0) { 1282 dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq); 1283 goto stop_irq_cap; 1284 } 1285 1286 cd->card_state = GENWQE_CARD_USED; 1287 return 0; 1288 1289 stop_irq_cap: 1290 genwqe_reset_interrupt_capability(cd); 1291 stop_kthread: 1292 kthread_stop(cd->card_thread); 1293 cd->card_thread = NULL; 1294 stop_free_queue: 1295 free_ddcb_queue(cd, queue); 1296 err_out: 1297 return rc; 1298 } 1299 1300 /** 1301 * queue_wake_up_all() - Handles fatal error case 1302 * 1303 * The PCI device got unusable and we have to stop all pending 1304 * requests as fast as we can. The code after this must purge the 1305 * DDCBs in question and ensure that all mappings are freed. 1306 */ 1307 static int queue_wake_up_all(struct genwqe_dev *cd) 1308 { 1309 unsigned int i; 1310 unsigned long flags; 1311 struct ddcb_queue *queue = &cd->queue; 1312 1313 spin_lock_irqsave(&queue->ddcb_lock, flags); 1314 1315 for (i = 0; i < queue->ddcb_max; i++) 1316 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]); 1317 1318 wake_up_interruptible(&queue->busy_waitq); 1319 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 1320 1321 return 0; 1322 } 1323 1324 /** 1325 * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces 1326 * 1327 * Relies on the pre-condition that there are no users of the card 1328 * device anymore e.g. with open file-descriptors. 1329 * 1330 * This function must be robust enough to be called twice. 1331 */ 1332 int genwqe_finish_queue(struct genwqe_dev *cd) 1333 { 1334 int i, rc = 0, in_flight; 1335 int waitmax = GENWQE_DDCB_SOFTWARE_TIMEOUT; 1336 struct pci_dev *pci_dev = cd->pci_dev; 1337 struct ddcb_queue *queue = &cd->queue; 1338 1339 if (!ddcb_queue_initialized(queue)) 1340 return 0; 1341 1342 /* Do not wipe out the error state. */ 1343 if (cd->card_state == GENWQE_CARD_USED) 1344 cd->card_state = GENWQE_CARD_UNUSED; 1345 1346 /* Wake up all requests in the DDCB queue such that they 1347 should be removed nicely. */ 1348 queue_wake_up_all(cd); 1349 1350 /* We must wait to get rid of the DDCBs in flight */ 1351 for (i = 0; i < waitmax; i++) { 1352 in_flight = genwqe_ddcbs_in_flight(cd); 1353 1354 if (in_flight == 0) 1355 break; 1356 1357 dev_dbg(&pci_dev->dev, 1358 " DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n", 1359 i, waitmax, in_flight); 1360 1361 /* 1362 * Severe severe error situation: The card itself has 1363 * 16 DDCB queues, each queue has e.g. 32 entries, 1364 * each DDBC has a hardware timeout of currently 250 1365 * msec but the PFs have a hardware timeout of 8 sec 1366 * ... so I take something large. 1367 */ 1368 msleep(1000); 1369 } 1370 if (i == waitmax) { 1371 dev_err(&pci_dev->dev, " [%s] err: queue is not empty!!\n", 1372 __func__); 1373 rc = -EIO; 1374 } 1375 return rc; 1376 } 1377 1378 /** 1379 * genwqe_release_service_layer() - Shutdown DDCB queue 1380 * @cd: genwqe device descriptor 1381 * 1382 * This function must be robust enough to be called twice. 1383 */ 1384 int genwqe_release_service_layer(struct genwqe_dev *cd) 1385 { 1386 struct pci_dev *pci_dev = cd->pci_dev; 1387 1388 if (!ddcb_queue_initialized(&cd->queue)) 1389 return 1; 1390 1391 free_irq(pci_dev->irq, cd); 1392 genwqe_reset_interrupt_capability(cd); 1393 1394 if (cd->card_thread != NULL) { 1395 kthread_stop(cd->card_thread); 1396 cd->card_thread = NULL; 1397 } 1398 1399 free_ddcb_queue(cd, &cd->queue); 1400 return 0; 1401 } 1402