1 /** 2 * IBM Accelerator Family 'GenWQE' 3 * 4 * (C) Copyright IBM Corp. 2013 5 * 6 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com> 7 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com> 8 * Author: Michael Jung <mijung@gmx.net> 9 * Author: Michael Ruettger <michael@ibmra.de> 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License (version 2 only) 13 * as published by the Free Software Foundation. 14 * 15 * This program is distributed in the hope that it will be useful, 16 * but WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 * GNU General Public License for more details. 19 */ 20 21 /* 22 * Device Driver Control Block (DDCB) queue support. Definition of 23 * interrupt handlers for queue support as well as triggering the 24 * health monitor code in case of problems. The current hardware uses 25 * an MSI interrupt which is shared between error handling and 26 * functional code. 27 */ 28 29 #include <linux/types.h> 30 #include <linux/module.h> 31 #include <linux/sched.h> 32 #include <linux/wait.h> 33 #include <linux/pci.h> 34 #include <linux/string.h> 35 #include <linux/dma-mapping.h> 36 #include <linux/delay.h> 37 #include <linux/module.h> 38 #include <linux/interrupt.h> 39 #include <linux/crc-itu-t.h> 40 41 #include "card_base.h" 42 #include "card_ddcb.h" 43 44 /* 45 * N: next DDCB, this is where the next DDCB will be put. 46 * A: active DDCB, this is where the code will look for the next completion. 47 * x: DDCB is enqueued, we are waiting for its completion. 48 49 * Situation (1): Empty queue 50 * +---+---+---+---+---+---+---+---+ 51 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 52 * | | | | | | | | | 53 * +---+---+---+---+---+---+---+---+ 54 * A/N 55 * enqueued_ddcbs = A - N = 2 - 2 = 0 56 * 57 * Situation (2): Wrapped, N > A 58 * +---+---+---+---+---+---+---+---+ 59 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 60 * | | | x | x | | | | | 61 * +---+---+---+---+---+---+---+---+ 62 * A N 63 * enqueued_ddcbs = N - A = 4 - 2 = 2 64 * 65 * Situation (3): Queue wrapped, A > N 66 * +---+---+---+---+---+---+---+---+ 67 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 68 * | x | x | | | x | x | x | x | 69 * +---+---+---+---+---+---+---+---+ 70 * N A 71 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 2) = 6 72 * 73 * Situation (4a): Queue full N > A 74 * +---+---+---+---+---+---+---+---+ 75 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 76 * | x | x | x | x | x | x | x | | 77 * +---+---+---+---+---+---+---+---+ 78 * A N 79 * 80 * enqueued_ddcbs = N - A = 7 - 0 = 7 81 * 82 * Situation (4a): Queue full A > N 83 * +---+---+---+---+---+---+---+---+ 84 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 85 * | x | x | x | | x | x | x | x | 86 * +---+---+---+---+---+---+---+---+ 87 * N A 88 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7 89 */ 90 91 static int queue_empty(struct ddcb_queue *queue) 92 { 93 return queue->ddcb_next == queue->ddcb_act; 94 } 95 96 static int queue_enqueued_ddcbs(struct ddcb_queue *queue) 97 { 98 if (queue->ddcb_next >= queue->ddcb_act) 99 return queue->ddcb_next - queue->ddcb_act; 100 101 return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next); 102 } 103 104 static int queue_free_ddcbs(struct ddcb_queue *queue) 105 { 106 int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1; 107 108 if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */ 109 return 0; 110 } 111 return free_ddcbs; 112 } 113 114 /* 115 * Use of the PRIV field in the DDCB for queue debugging: 116 * 117 * (1) Trying to get rid of a DDCB which saw a timeout: 118 * pddcb->priv[6] = 0xcc; # cleared 119 * 120 * (2) Append a DDCB via NEXT bit: 121 * pddcb->priv[7] = 0xaa; # appended 122 * 123 * (3) DDCB needed tapping: 124 * pddcb->priv[7] = 0xbb; # tapped 125 * 126 * (4) DDCB marked as correctly finished: 127 * pddcb->priv[6] = 0xff; # finished 128 */ 129 130 static inline void ddcb_mark_tapped(struct ddcb *pddcb) 131 { 132 pddcb->priv[7] = 0xbb; /* tapped */ 133 } 134 135 static inline void ddcb_mark_appended(struct ddcb *pddcb) 136 { 137 pddcb->priv[7] = 0xaa; /* appended */ 138 } 139 140 static inline void ddcb_mark_cleared(struct ddcb *pddcb) 141 { 142 pddcb->priv[6] = 0xcc; /* cleared */ 143 } 144 145 static inline void ddcb_mark_finished(struct ddcb *pddcb) 146 { 147 pddcb->priv[6] = 0xff; /* finished */ 148 } 149 150 static inline void ddcb_mark_unused(struct ddcb *pddcb) 151 { 152 pddcb->priv_64 = cpu_to_be64(0); /* not tapped */ 153 } 154 155 /** 156 * genwqe_crc16() - Generate 16-bit crc as required for DDCBs 157 * @buff: pointer to data buffer 158 * @len: length of data for calculation 159 * @init: initial crc (0xffff at start) 160 * 161 * Polynomial = x^16 + x^12 + x^5 + 1 (0x1021) 162 * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff 163 * should result in a crc16 of 0x89c3 164 * 165 * Return: crc16 checksum in big endian format ! 166 */ 167 static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init) 168 { 169 return crc_itu_t(init, buff, len); 170 } 171 172 static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue) 173 { 174 int i; 175 struct ddcb *pddcb; 176 unsigned long flags; 177 struct pci_dev *pci_dev = cd->pci_dev; 178 179 spin_lock_irqsave(&cd->print_lock, flags); 180 181 dev_info(&pci_dev->dev, 182 "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n", 183 cd->card_idx, queue->ddcb_act, queue->ddcb_next); 184 185 pddcb = queue->ddcb_vaddr; 186 for (i = 0; i < queue->ddcb_max; i++) { 187 dev_err(&pci_dev->dev, 188 " %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n", 189 i == queue->ddcb_act ? '>' : ' ', 190 i, 191 be16_to_cpu(pddcb->retc_16), 192 be16_to_cpu(pddcb->seqnum_16), 193 pddcb->hsi, 194 pddcb->shi, 195 be64_to_cpu(pddcb->priv_64), 196 pddcb->cmd); 197 pddcb++; 198 } 199 spin_unlock_irqrestore(&cd->print_lock, flags); 200 } 201 202 struct genwqe_ddcb_cmd *ddcb_requ_alloc(void) 203 { 204 struct ddcb_requ *req; 205 206 req = kzalloc(sizeof(*req), GFP_KERNEL); 207 if (!req) 208 return NULL; 209 210 return &req->cmd; 211 } 212 213 void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd) 214 { 215 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd); 216 217 kfree(req); 218 } 219 220 static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req) 221 { 222 return req->req_state; 223 } 224 225 static inline void ddcb_requ_set_state(struct ddcb_requ *req, 226 enum genwqe_requ_state new_state) 227 { 228 req->req_state = new_state; 229 } 230 231 static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req) 232 { 233 return req->cmd.ddata_addr != 0x0; 234 } 235 236 /** 237 * ddcb_requ_finished() - Returns the hardware state of the associated DDCB 238 * @cd: pointer to genwqe device descriptor 239 * @req: DDCB work request 240 * 241 * Status of ddcb_requ mirrors this hardware state, but is copied in 242 * the ddcb_requ on interrupt/polling function. The lowlevel code 243 * should check the hardware state directly, the higher level code 244 * should check the copy. 245 * 246 * This function will also return true if the state of the queue is 247 * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the 248 * shutdown case. 249 */ 250 static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req) 251 { 252 return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) || 253 (cd->card_state != GENWQE_CARD_USED); 254 } 255 256 /** 257 * enqueue_ddcb() - Enqueue a DDCB 258 * @cd: pointer to genwqe device descriptor 259 * @queue: queue this operation should be done on 260 * @ddcb_no: pointer to ddcb number being tapped 261 * 262 * Start execution of DDCB by tapping or append to queue via NEXT 263 * bit. This is done by an atomic 'compare and swap' instruction and 264 * checking SHI and HSI of the previous DDCB. 265 * 266 * This function must only be called with ddcb_lock held. 267 * 268 * Return: 1 if new DDCB is appended to previous 269 * 2 if DDCB queue is tapped via register/simulation 270 */ 271 #define RET_DDCB_APPENDED 1 272 #define RET_DDCB_TAPPED 2 273 274 static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue, 275 struct ddcb *pddcb, int ddcb_no) 276 { 277 unsigned int try; 278 int prev_no; 279 struct ddcb *prev_ddcb; 280 __be32 old, new, icrc_hsi_shi; 281 u64 num; 282 283 /* 284 * For performance checks a Dispatch Timestamp can be put into 285 * DDCB It is supposed to use the SLU's free running counter, 286 * but this requires PCIe cycles. 287 */ 288 ddcb_mark_unused(pddcb); 289 290 /* check previous DDCB if already fetched */ 291 prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1; 292 prev_ddcb = &queue->ddcb_vaddr[prev_no]; 293 294 /* 295 * It might have happened that the HSI.FETCHED bit is 296 * set. Retry in this case. Therefore I expect maximum 2 times 297 * trying. 298 */ 299 ddcb_mark_appended(pddcb); 300 for (try = 0; try < 2; try++) { 301 old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */ 302 303 /* try to append via NEXT bit if prev DDCB is not completed */ 304 if ((old & DDCB_COMPLETED_BE32) != 0x00000000) 305 break; 306 307 new = (old | DDCB_NEXT_BE32); 308 309 wmb(); /* need to ensure write ordering */ 310 icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new); 311 312 if (icrc_hsi_shi == old) 313 return RET_DDCB_APPENDED; /* appended to queue */ 314 } 315 316 /* Queue must be re-started by updating QUEUE_OFFSET */ 317 ddcb_mark_tapped(pddcb); 318 num = (u64)ddcb_no << 8; 319 320 wmb(); /* need to ensure write ordering */ 321 __genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */ 322 323 return RET_DDCB_TAPPED; 324 } 325 326 /** 327 * copy_ddcb_results() - Copy output state from real DDCB to request 328 * 329 * Copy DDCB ASV to request struct. There is no endian 330 * conversion made, since data structure in ASV is still 331 * unknown here. 332 * 333 * This is needed by: 334 * - genwqe_purge_ddcb() 335 * - genwqe_check_ddcb_queue() 336 */ 337 static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no) 338 { 339 struct ddcb_queue *queue = req->queue; 340 struct ddcb *pddcb = &queue->ddcb_vaddr[req->num]; 341 342 memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH); 343 344 /* copy status flags of the variant part */ 345 req->cmd.vcrc = be16_to_cpu(pddcb->vcrc_16); 346 req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64); 347 req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64); 348 349 req->cmd.attn = be16_to_cpu(pddcb->attn_16); 350 req->cmd.progress = be32_to_cpu(pddcb->progress_32); 351 req->cmd.retc = be16_to_cpu(pddcb->retc_16); 352 353 if (ddcb_requ_collect_debug_data(req)) { 354 int prev_no = (ddcb_no == 0) ? 355 queue->ddcb_max - 1 : ddcb_no - 1; 356 struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no]; 357 358 memcpy(&req->debug_data.ddcb_finished, pddcb, 359 sizeof(req->debug_data.ddcb_finished)); 360 memcpy(&req->debug_data.ddcb_prev, prev_pddcb, 361 sizeof(req->debug_data.ddcb_prev)); 362 } 363 } 364 365 /** 366 * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work equests. 367 * @cd: pointer to genwqe device descriptor 368 * 369 * Return: Number of DDCBs which were finished 370 */ 371 static int genwqe_check_ddcb_queue(struct genwqe_dev *cd, 372 struct ddcb_queue *queue) 373 { 374 unsigned long flags; 375 int ddcbs_finished = 0; 376 struct pci_dev *pci_dev = cd->pci_dev; 377 378 spin_lock_irqsave(&queue->ddcb_lock, flags); 379 380 /* FIXME avoid soft locking CPU */ 381 while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) { 382 383 struct ddcb *pddcb; 384 struct ddcb_requ *req; 385 u16 vcrc, vcrc_16, retc_16; 386 387 pddcb = &queue->ddcb_vaddr[queue->ddcb_act]; 388 389 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 390 0x00000000) 391 goto go_home; /* not completed, continue waiting */ 392 393 wmb(); /* Add sync to decouple prev. read operations */ 394 395 /* Note: DDCB could be purged */ 396 req = queue->ddcb_req[queue->ddcb_act]; 397 if (req == NULL) { 398 /* this occurs if DDCB is purged, not an error */ 399 /* Move active DDCB further; Nothing to do anymore. */ 400 goto pick_next_one; 401 } 402 403 /* 404 * HSI=0x44 (fetched and completed), but RETC is 405 * 0x101, or even worse 0x000. 406 * 407 * In case of seeing the queue in inconsistent state 408 * we read the errcnts and the queue status to provide 409 * a trigger for our PCIe analyzer stop capturing. 410 */ 411 retc_16 = be16_to_cpu(pddcb->retc_16); 412 if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) { 413 u64 errcnts, status; 414 u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr; 415 416 errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS); 417 status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS); 418 419 dev_err(&pci_dev->dev, 420 "[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n", 421 __func__, be16_to_cpu(pddcb->seqnum_16), 422 pddcb->hsi, retc_16, errcnts, status, 423 queue->ddcb_daddr + ddcb_offs); 424 } 425 426 copy_ddcb_results(req, queue->ddcb_act); 427 queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */ 428 429 dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num); 430 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); 431 432 ddcb_mark_finished(pddcb); 433 434 /* calculate CRC_16 to see if VCRC is correct */ 435 vcrc = genwqe_crc16(pddcb->asv, 436 VCRC_LENGTH(req->cmd.asv_length), 437 0xffff); 438 vcrc_16 = be16_to_cpu(pddcb->vcrc_16); 439 if (vcrc != vcrc_16) { 440 printk_ratelimited(KERN_ERR 441 "%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n", 442 GENWQE_DEVNAME, dev_name(&pci_dev->dev), 443 pddcb->pre, VCRC_LENGTH(req->cmd.asv_length), 444 vcrc, vcrc_16); 445 } 446 447 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED); 448 queue->ddcbs_completed++; 449 queue->ddcbs_in_flight--; 450 451 /* wake up process waiting for this DDCB, and 452 processes on the busy queue */ 453 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]); 454 wake_up_interruptible(&queue->busy_waitq); 455 456 pick_next_one: 457 queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max; 458 ddcbs_finished++; 459 } 460 461 go_home: 462 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 463 return ddcbs_finished; 464 } 465 466 /** 467 * __genwqe_wait_ddcb(): Waits until DDCB is completed 468 * @cd: pointer to genwqe device descriptor 469 * @req: pointer to requsted DDCB parameters 470 * 471 * The Service Layer will update the RETC in DDCB when processing is 472 * pending or done. 473 * 474 * Return: > 0 remaining jiffies, DDCB completed 475 * -ETIMEDOUT when timeout 476 * -ERESTARTSYS when ^C 477 * -EINVAL when unknown error condition 478 * 479 * When an error is returned the called needs to ensure that 480 * purge_ddcb() is being called to get the &req removed from the 481 * queue. 482 */ 483 int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req) 484 { 485 int rc; 486 unsigned int ddcb_no; 487 struct ddcb_queue *queue; 488 struct pci_dev *pci_dev = cd->pci_dev; 489 490 if (req == NULL) 491 return -EINVAL; 492 493 queue = req->queue; 494 if (queue == NULL) 495 return -EINVAL; 496 497 ddcb_no = req->num; 498 if (ddcb_no >= queue->ddcb_max) 499 return -EINVAL; 500 501 rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no], 502 ddcb_requ_finished(cd, req), 503 GENWQE_DDCB_SOFTWARE_TIMEOUT * HZ); 504 505 /* 506 * We need to distinguish 3 cases here: 507 * 1. rc == 0 timeout occured 508 * 2. rc == -ERESTARTSYS signal received 509 * 3. rc > 0 remaining jiffies condition is true 510 */ 511 if (rc == 0) { 512 struct ddcb_queue *queue = req->queue; 513 struct ddcb *pddcb; 514 515 /* 516 * Timeout may be caused by long task switching time. 517 * When timeout happens, check if the request has 518 * meanwhile completed. 519 */ 520 genwqe_check_ddcb_queue(cd, req->queue); 521 if (ddcb_requ_finished(cd, req)) 522 return rc; 523 524 dev_err(&pci_dev->dev, 525 "[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n", 526 __func__, req->num, rc, ddcb_requ_get_state(req), 527 req); 528 dev_err(&pci_dev->dev, 529 "[%s] IO_QUEUE_STATUS=0x%016llx\n", __func__, 530 __genwqe_readq(cd, queue->IO_QUEUE_STATUS)); 531 532 pddcb = &queue->ddcb_vaddr[req->num]; 533 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); 534 535 print_ddcb_info(cd, req->queue); 536 return -ETIMEDOUT; 537 538 } else if (rc == -ERESTARTSYS) { 539 return rc; 540 /* 541 * EINTR: Stops the application 542 * ERESTARTSYS: Restartable systemcall; called again 543 */ 544 545 } else if (rc < 0) { 546 dev_err(&pci_dev->dev, 547 "[%s] err: DDCB#%d unknown result (rc=%d) %d!\n", 548 __func__, req->num, rc, ddcb_requ_get_state(req)); 549 return -EINVAL; 550 } 551 552 /* Severe error occured. Driver is forced to stop operation */ 553 if (cd->card_state != GENWQE_CARD_USED) { 554 dev_err(&pci_dev->dev, 555 "[%s] err: DDCB#%d forced to stop (rc=%d)\n", 556 __func__, req->num, rc); 557 return -EIO; 558 } 559 return rc; 560 } 561 562 /** 563 * get_next_ddcb() - Get next available DDCB 564 * @cd: pointer to genwqe device descriptor 565 * 566 * DDCB's content is completely cleared but presets for PRE and 567 * SEQNUM. This function must only be called when ddcb_lock is held. 568 * 569 * Return: NULL if no empty DDCB available otherwise ptr to next DDCB. 570 */ 571 static struct ddcb *get_next_ddcb(struct genwqe_dev *cd, 572 struct ddcb_queue *queue, 573 int *num) 574 { 575 u64 *pu64; 576 struct ddcb *pddcb; 577 578 if (queue_free_ddcbs(queue) == 0) /* queue is full */ 579 return NULL; 580 581 /* find new ddcb */ 582 pddcb = &queue->ddcb_vaddr[queue->ddcb_next]; 583 584 /* if it is not completed, we are not allowed to use it */ 585 /* barrier(); */ 586 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000) 587 return NULL; 588 589 *num = queue->ddcb_next; /* internal DDCB number */ 590 queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max; 591 592 /* clear important DDCB fields */ 593 pu64 = (u64 *)pddcb; 594 pu64[0] = 0ULL; /* offs 0x00 (ICRC,HSI,SHI,...) */ 595 pu64[1] = 0ULL; /* offs 0x01 (ACFUNC,CMD...) */ 596 597 /* destroy previous results in ASV */ 598 pu64[0x80/8] = 0ULL; /* offs 0x80 (ASV + 0) */ 599 pu64[0x88/8] = 0ULL; /* offs 0x88 (ASV + 0x08) */ 600 pu64[0x90/8] = 0ULL; /* offs 0x90 (ASV + 0x10) */ 601 pu64[0x98/8] = 0ULL; /* offs 0x98 (ASV + 0x18) */ 602 pu64[0xd0/8] = 0ULL; /* offs 0xd0 (RETC,ATTN...) */ 603 604 pddcb->pre = DDCB_PRESET_PRE; /* 128 */ 605 pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++); 606 return pddcb; 607 } 608 609 /** 610 * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue 611 * @cd: genwqe device descriptor 612 * @req: DDCB request 613 * 614 * This will fail when the request was already FETCHED. In this case 615 * we need to wait until it is finished. Else the DDCB can be 616 * reused. This function also ensures that the request data structure 617 * is removed from ddcb_req[]. 618 * 619 * Do not forget to call this function when genwqe_wait_ddcb() fails, 620 * such that the request gets really removed from ddcb_req[]. 621 * 622 * Return: 0 success 623 */ 624 int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req) 625 { 626 struct ddcb *pddcb = NULL; 627 unsigned int t; 628 unsigned long flags; 629 struct ddcb_queue *queue = req->queue; 630 struct pci_dev *pci_dev = cd->pci_dev; 631 u64 queue_status; 632 __be32 icrc_hsi_shi = 0x0000; 633 __be32 old, new; 634 635 /* unsigned long flags; */ 636 if (GENWQE_DDCB_SOFTWARE_TIMEOUT <= 0) { 637 dev_err(&pci_dev->dev, 638 "[%s] err: software timeout is not set!\n", __func__); 639 return -EFAULT; 640 } 641 642 pddcb = &queue->ddcb_vaddr[req->num]; 643 644 for (t = 0; t < GENWQE_DDCB_SOFTWARE_TIMEOUT * 10; t++) { 645 646 spin_lock_irqsave(&queue->ddcb_lock, flags); 647 648 /* Check if req was meanwhile finished */ 649 if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) 650 goto go_home; 651 652 /* try to set PURGE bit if FETCHED/COMPLETED are not set */ 653 old = pddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */ 654 if ((old & DDCB_FETCHED_BE32) == 0x00000000) { 655 656 new = (old | DDCB_PURGE_BE32); 657 icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32, 658 old, new); 659 if (icrc_hsi_shi == old) 660 goto finish_ddcb; 661 } 662 663 /* normal finish with HSI bit */ 664 barrier(); 665 icrc_hsi_shi = pddcb->icrc_hsi_shi_32; 666 if (icrc_hsi_shi & DDCB_COMPLETED_BE32) 667 goto finish_ddcb; 668 669 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 670 671 /* 672 * Here the check_ddcb() function will most likely 673 * discover this DDCB to be finished some point in 674 * time. It will mark the req finished and free it up 675 * in the list. 676 */ 677 678 copy_ddcb_results(req, req->num); /* for the failing case */ 679 msleep(100); /* sleep for 1/10 second and try again */ 680 continue; 681 682 finish_ddcb: 683 copy_ddcb_results(req, req->num); 684 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED); 685 queue->ddcbs_in_flight--; 686 queue->ddcb_req[req->num] = NULL; /* delete from array */ 687 ddcb_mark_cleared(pddcb); 688 689 /* Move active DDCB further; Nothing to do here anymore. */ 690 691 /* 692 * We need to ensure that there is at least one free 693 * DDCB in the queue. To do that, we must update 694 * ddcb_act only if the COMPLETED bit is set for the 695 * DDCB we are working on else we treat that DDCB even 696 * if we PURGED it as occupied (hardware is supposed 697 * to set the COMPLETED bit yet!). 698 */ 699 icrc_hsi_shi = pddcb->icrc_hsi_shi_32; 700 if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) && 701 (queue->ddcb_act == req->num)) { 702 queue->ddcb_act = ((queue->ddcb_act + 1) % 703 queue->ddcb_max); 704 } 705 go_home: 706 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 707 return 0; 708 } 709 710 /* 711 * If the card is dead and the queue is forced to stop, we 712 * might see this in the queue status register. 713 */ 714 queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS); 715 716 dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num); 717 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); 718 719 dev_err(&pci_dev->dev, 720 "[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n", 721 __func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT, 722 queue_status); 723 724 print_ddcb_info(cd, req->queue); 725 726 return -EFAULT; 727 } 728 729 int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d) 730 { 731 int len; 732 struct pci_dev *pci_dev = cd->pci_dev; 733 734 if (d == NULL) { 735 dev_err(&pci_dev->dev, 736 "[%s] err: invalid memory for debug data!\n", 737 __func__); 738 return -EFAULT; 739 } 740 741 len = sizeof(d->driver_version); 742 snprintf(d->driver_version, len, "%s", DRV_VERSION); 743 d->slu_unitcfg = cd->slu_unitcfg; 744 d->app_unitcfg = cd->app_unitcfg; 745 return 0; 746 } 747 748 /** 749 * __genwqe_enqueue_ddcb() - Enqueue a DDCB 750 * @cd: pointer to genwqe device descriptor 751 * @req: pointer to DDCB execution request 752 * @f_flags: file mode: blocking, non-blocking 753 * 754 * Return: 0 if enqueuing succeeded 755 * -EIO if card is unusable/PCIe problems 756 * -EBUSY if enqueuing failed 757 */ 758 int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req, 759 unsigned int f_flags) 760 { 761 struct ddcb *pddcb; 762 unsigned long flags; 763 struct ddcb_queue *queue; 764 struct pci_dev *pci_dev = cd->pci_dev; 765 u16 icrc; 766 767 retry: 768 if (cd->card_state != GENWQE_CARD_USED) { 769 printk_ratelimited(KERN_ERR 770 "%s %s: [%s] Card is unusable/PCIe problem Req#%d\n", 771 GENWQE_DEVNAME, dev_name(&pci_dev->dev), 772 __func__, req->num); 773 return -EIO; 774 } 775 776 queue = req->queue = &cd->queue; 777 778 /* FIXME circumvention to improve performance when no irq is 779 * there. 780 */ 781 if (GENWQE_POLLING_ENABLED) 782 genwqe_check_ddcb_queue(cd, queue); 783 784 /* 785 * It must be ensured to process all DDCBs in successive 786 * order. Use a lock here in order to prevent nested DDCB 787 * enqueuing. 788 */ 789 spin_lock_irqsave(&queue->ddcb_lock, flags); 790 791 pddcb = get_next_ddcb(cd, queue, &req->num); /* get ptr and num */ 792 if (pddcb == NULL) { 793 int rc; 794 795 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 796 797 if (f_flags & O_NONBLOCK) { 798 queue->return_on_busy++; 799 return -EBUSY; 800 } 801 802 queue->wait_on_busy++; 803 rc = wait_event_interruptible(queue->busy_waitq, 804 queue_free_ddcbs(queue) != 0); 805 dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n", 806 __func__, rc); 807 if (rc == -ERESTARTSYS) 808 return rc; /* interrupted by a signal */ 809 810 goto retry; 811 } 812 813 if (queue->ddcb_req[req->num] != NULL) { 814 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 815 816 dev_err(&pci_dev->dev, 817 "[%s] picked DDCB %d with req=%p still in use!!\n", 818 __func__, req->num, req); 819 return -EFAULT; 820 } 821 ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED); 822 queue->ddcb_req[req->num] = req; 823 824 pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts); 825 pddcb->cmd = req->cmd.cmd; 826 pddcb->acfunc = req->cmd.acfunc; /* functional unit */ 827 828 /* 829 * We know that we can get retc 0x104 with CRC error, do not 830 * stop the queue in those cases for this command. XDIR = 1 831 * does not work for old SLU versions. 832 * 833 * Last bitstream with the old XDIR behavior had SLU_ID 834 * 0x34199. 835 */ 836 if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull) 837 pddcb->xdir = 0x1; 838 else 839 pddcb->xdir = 0x0; 840 841 842 pddcb->psp = (((req->cmd.asiv_length / 8) << 4) | 843 ((req->cmd.asv_length / 8))); 844 pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts); 845 846 /* 847 * If copying the whole DDCB_ASIV_LENGTH is impacting 848 * performance we need to change it to 849 * req->cmd.asiv_length. But simulation benefits from some 850 * non-architectured bits behind the architectured content. 851 * 852 * How much data is copied depends on the availability of the 853 * ATS field, which was introduced late. If the ATS field is 854 * supported ASIV is 8 bytes shorter than it used to be. Since 855 * the ATS field is copied too, the code should do exactly 856 * what it did before, but I wanted to make copying of the ATS 857 * field very explicit. 858 */ 859 if (genwqe_get_slu_id(cd) <= 0x2) { 860 memcpy(&pddcb->__asiv[0], /* destination */ 861 &req->cmd.__asiv[0], /* source */ 862 DDCB_ASIV_LENGTH); /* req->cmd.asiv_length */ 863 } else { 864 pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats); 865 memcpy(&pddcb->n.asiv[0], /* destination */ 866 &req->cmd.asiv[0], /* source */ 867 DDCB_ASIV_LENGTH_ATS); /* req->cmd.asiv_length */ 868 } 869 870 pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */ 871 872 /* 873 * Calculate CRC_16 for corresponding range PSP(7:4). Include 874 * empty 4 bytes prior to the data. 875 */ 876 icrc = genwqe_crc16((const u8 *)pddcb, 877 ICRC_LENGTH(req->cmd.asiv_length), 0xffff); 878 pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16); 879 880 /* enable DDCB completion irq */ 881 if (!GENWQE_POLLING_ENABLED) 882 pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32; 883 884 dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num); 885 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); 886 887 if (ddcb_requ_collect_debug_data(req)) { 888 /* use the kernel copy of debug data. copying back to 889 user buffer happens later */ 890 891 genwqe_init_debug_data(cd, &req->debug_data); 892 memcpy(&req->debug_data.ddcb_before, pddcb, 893 sizeof(req->debug_data.ddcb_before)); 894 } 895 896 enqueue_ddcb(cd, queue, pddcb, req->num); 897 queue->ddcbs_in_flight++; 898 899 if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight) 900 queue->ddcbs_max_in_flight = queue->ddcbs_in_flight; 901 902 ddcb_requ_set_state(req, GENWQE_REQU_TAPPED); 903 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 904 wake_up_interruptible(&cd->queue_waitq); 905 906 return 0; 907 } 908 909 /** 910 * __genwqe_execute_raw_ddcb() - Setup and execute DDCB 911 * @cd: pointer to genwqe device descriptor 912 * @req: user provided DDCB request 913 * @f_flags: file mode: blocking, non-blocking 914 */ 915 int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd, 916 struct genwqe_ddcb_cmd *cmd, 917 unsigned int f_flags) 918 { 919 int rc = 0; 920 struct pci_dev *pci_dev = cd->pci_dev; 921 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd); 922 923 if (cmd->asiv_length > DDCB_ASIV_LENGTH) { 924 dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n", 925 __func__, cmd->asiv_length); 926 return -EINVAL; 927 } 928 if (cmd->asv_length > DDCB_ASV_LENGTH) { 929 dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n", 930 __func__, cmd->asiv_length); 931 return -EINVAL; 932 } 933 rc = __genwqe_enqueue_ddcb(cd, req, f_flags); 934 if (rc != 0) 935 return rc; 936 937 rc = __genwqe_wait_ddcb(cd, req); 938 if (rc < 0) /* error or signal interrupt */ 939 goto err_exit; 940 941 if (ddcb_requ_collect_debug_data(req)) { 942 if (copy_to_user((struct genwqe_debug_data __user *) 943 (unsigned long)cmd->ddata_addr, 944 &req->debug_data, 945 sizeof(struct genwqe_debug_data))) 946 return -EFAULT; 947 } 948 949 /* 950 * Higher values than 0x102 indicate completion with faults, 951 * lower values than 0x102 indicate processing faults. Note 952 * that DDCB might have been purged. E.g. Cntl+C. 953 */ 954 if (cmd->retc != DDCB_RETC_COMPLETE) { 955 /* This might happen e.g. flash read, and needs to be 956 handled by the upper layer code. */ 957 rc = -EBADMSG; /* not processed/error retc */ 958 } 959 960 return rc; 961 962 err_exit: 963 __genwqe_purge_ddcb(cd, req); 964 965 if (ddcb_requ_collect_debug_data(req)) { 966 if (copy_to_user((struct genwqe_debug_data __user *) 967 (unsigned long)cmd->ddata_addr, 968 &req->debug_data, 969 sizeof(struct genwqe_debug_data))) 970 return -EFAULT; 971 } 972 return rc; 973 } 974 975 /** 976 * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished 977 * 978 * We use this as condition for our wait-queue code. 979 */ 980 static int genwqe_next_ddcb_ready(struct genwqe_dev *cd) 981 { 982 unsigned long flags; 983 struct ddcb *pddcb; 984 struct ddcb_queue *queue = &cd->queue; 985 986 spin_lock_irqsave(&queue->ddcb_lock, flags); 987 988 if (queue_empty(queue)) { /* emtpy queue */ 989 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 990 return 0; 991 } 992 993 pddcb = &queue->ddcb_vaddr[queue->ddcb_act]; 994 if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */ 995 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 996 return 1; 997 } 998 999 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 1000 return 0; 1001 } 1002 1003 /** 1004 * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight 1005 * 1006 * Keep track on the number of DDCBs which ware currently in the 1007 * queue. This is needed for statistics as well as conditon if we want 1008 * to wait or better do polling in case of no interrupts available. 1009 */ 1010 int genwqe_ddcbs_in_flight(struct genwqe_dev *cd) 1011 { 1012 unsigned long flags; 1013 int ddcbs_in_flight = 0; 1014 struct ddcb_queue *queue = &cd->queue; 1015 1016 spin_lock_irqsave(&queue->ddcb_lock, flags); 1017 ddcbs_in_flight += queue->ddcbs_in_flight; 1018 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 1019 1020 return ddcbs_in_flight; 1021 } 1022 1023 static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue) 1024 { 1025 int rc, i; 1026 struct ddcb *pddcb; 1027 u64 val64; 1028 unsigned int queue_size; 1029 struct pci_dev *pci_dev = cd->pci_dev; 1030 1031 if (GENWQE_DDCB_MAX < 2) 1032 return -EINVAL; 1033 1034 queue_size = roundup(GENWQE_DDCB_MAX * sizeof(struct ddcb), PAGE_SIZE); 1035 1036 queue->ddcbs_in_flight = 0; /* statistics */ 1037 queue->ddcbs_max_in_flight = 0; 1038 queue->ddcbs_completed = 0; 1039 queue->return_on_busy = 0; 1040 queue->wait_on_busy = 0; 1041 1042 queue->ddcb_seq = 0x100; /* start sequence number */ 1043 queue->ddcb_max = GENWQE_DDCB_MAX; 1044 queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size, 1045 &queue->ddcb_daddr); 1046 if (queue->ddcb_vaddr == NULL) { 1047 dev_err(&pci_dev->dev, 1048 "[%s] **err: could not allocate DDCB **\n", __func__); 1049 return -ENOMEM; 1050 } 1051 queue->ddcb_req = kzalloc(sizeof(struct ddcb_requ *) * 1052 queue->ddcb_max, GFP_KERNEL); 1053 if (!queue->ddcb_req) { 1054 rc = -ENOMEM; 1055 goto free_ddcbs; 1056 } 1057 1058 queue->ddcb_waitqs = kzalloc(sizeof(wait_queue_head_t) * 1059 queue->ddcb_max, GFP_KERNEL); 1060 if (!queue->ddcb_waitqs) { 1061 rc = -ENOMEM; 1062 goto free_requs; 1063 } 1064 1065 for (i = 0; i < queue->ddcb_max; i++) { 1066 pddcb = &queue->ddcb_vaddr[i]; /* DDCBs */ 1067 pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32; 1068 pddcb->retc_16 = cpu_to_be16(0xfff); 1069 1070 queue->ddcb_req[i] = NULL; /* requests */ 1071 init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */ 1072 } 1073 1074 queue->ddcb_act = 0; 1075 queue->ddcb_next = 0; /* queue is empty */ 1076 1077 spin_lock_init(&queue->ddcb_lock); 1078 init_waitqueue_head(&queue->busy_waitq); 1079 1080 val64 = ((u64)(queue->ddcb_max - 1) << 8); /* lastptr */ 1081 __genwqe_writeq(cd, queue->IO_QUEUE_CONFIG, 0x07); /* iCRC/vCRC */ 1082 __genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr); 1083 __genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq); 1084 __genwqe_writeq(cd, queue->IO_QUEUE_WRAP, val64); 1085 return 0; 1086 1087 free_requs: 1088 kfree(queue->ddcb_req); 1089 queue->ddcb_req = NULL; 1090 free_ddcbs: 1091 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr, 1092 queue->ddcb_daddr); 1093 queue->ddcb_vaddr = NULL; 1094 queue->ddcb_daddr = 0ull; 1095 return -ENODEV; 1096 1097 } 1098 1099 static int ddcb_queue_initialized(struct ddcb_queue *queue) 1100 { 1101 return queue->ddcb_vaddr != NULL; 1102 } 1103 1104 static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue) 1105 { 1106 unsigned int queue_size; 1107 1108 queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE); 1109 1110 kfree(queue->ddcb_req); 1111 queue->ddcb_req = NULL; 1112 1113 if (queue->ddcb_vaddr) { 1114 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr, 1115 queue->ddcb_daddr); 1116 queue->ddcb_vaddr = NULL; 1117 queue->ddcb_daddr = 0ull; 1118 } 1119 } 1120 1121 static irqreturn_t genwqe_pf_isr(int irq, void *dev_id) 1122 { 1123 u64 gfir; 1124 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id; 1125 struct pci_dev *pci_dev = cd->pci_dev; 1126 1127 /* 1128 * In case of fatal FIR error the queue is stopped, such that 1129 * we can safely check it without risking anything. 1130 */ 1131 cd->irqs_processed++; 1132 wake_up_interruptible(&cd->queue_waitq); 1133 1134 /* 1135 * Checking for errors before kicking the queue might be 1136 * safer, but slower for the good-case ... See above. 1137 */ 1138 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR); 1139 if (((gfir & GFIR_ERR_TRIGGER) != 0x0) && 1140 !pci_channel_offline(pci_dev)) { 1141 1142 if (cd->use_platform_recovery) { 1143 /* 1144 * Since we use raw accessors, EEH errors won't be 1145 * detected by the platform until we do a non-raw 1146 * MMIO or config space read 1147 */ 1148 readq(cd->mmio + IO_SLC_CFGREG_GFIR); 1149 1150 /* Don't do anything if the PCI channel is frozen */ 1151 if (pci_channel_offline(pci_dev)) 1152 goto exit; 1153 } 1154 1155 wake_up_interruptible(&cd->health_waitq); 1156 1157 /* 1158 * By default GFIRs causes recovery actions. This 1159 * count is just for debug when recovery is masked. 1160 */ 1161 dev_err_ratelimited(&pci_dev->dev, 1162 "[%s] GFIR=%016llx\n", 1163 __func__, gfir); 1164 } 1165 1166 exit: 1167 return IRQ_HANDLED; 1168 } 1169 1170 static irqreturn_t genwqe_vf_isr(int irq, void *dev_id) 1171 { 1172 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id; 1173 1174 cd->irqs_processed++; 1175 wake_up_interruptible(&cd->queue_waitq); 1176 1177 return IRQ_HANDLED; 1178 } 1179 1180 /** 1181 * genwqe_card_thread() - Work thread for the DDCB queue 1182 * 1183 * The idea is to check if there are DDCBs in processing. If there are 1184 * some finished DDCBs, we process them and wakeup the 1185 * requestors. Otherwise we give other processes time using 1186 * cond_resched(). 1187 */ 1188 static int genwqe_card_thread(void *data) 1189 { 1190 int should_stop = 0, rc = 0; 1191 struct genwqe_dev *cd = (struct genwqe_dev *)data; 1192 1193 while (!kthread_should_stop()) { 1194 1195 genwqe_check_ddcb_queue(cd, &cd->queue); 1196 1197 if (GENWQE_POLLING_ENABLED) { 1198 rc = wait_event_interruptible_timeout( 1199 cd->queue_waitq, 1200 genwqe_ddcbs_in_flight(cd) || 1201 (should_stop = kthread_should_stop()), 1); 1202 } else { 1203 rc = wait_event_interruptible_timeout( 1204 cd->queue_waitq, 1205 genwqe_next_ddcb_ready(cd) || 1206 (should_stop = kthread_should_stop()), HZ); 1207 } 1208 if (should_stop) 1209 break; 1210 1211 /* 1212 * Avoid soft lockups on heavy loads; we do not want 1213 * to disable our interrupts. 1214 */ 1215 cond_resched(); 1216 } 1217 return 0; 1218 } 1219 1220 /** 1221 * genwqe_setup_service_layer() - Setup DDCB queue 1222 * @cd: pointer to genwqe device descriptor 1223 * 1224 * Allocate DDCBs. Configure Service Layer Controller (SLC). 1225 * 1226 * Return: 0 success 1227 */ 1228 int genwqe_setup_service_layer(struct genwqe_dev *cd) 1229 { 1230 int rc; 1231 struct ddcb_queue *queue; 1232 struct pci_dev *pci_dev = cd->pci_dev; 1233 1234 if (genwqe_is_privileged(cd)) { 1235 rc = genwqe_card_reset(cd); 1236 if (rc < 0) { 1237 dev_err(&pci_dev->dev, 1238 "[%s] err: reset failed.\n", __func__); 1239 return rc; 1240 } 1241 genwqe_read_softreset(cd); 1242 } 1243 1244 queue = &cd->queue; 1245 queue->IO_QUEUE_CONFIG = IO_SLC_QUEUE_CONFIG; 1246 queue->IO_QUEUE_STATUS = IO_SLC_QUEUE_STATUS; 1247 queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT; 1248 queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN; 1249 queue->IO_QUEUE_OFFSET = IO_SLC_QUEUE_OFFSET; 1250 queue->IO_QUEUE_WRAP = IO_SLC_QUEUE_WRAP; 1251 queue->IO_QUEUE_WTIME = IO_SLC_QUEUE_WTIME; 1252 queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS; 1253 queue->IO_QUEUE_LRW = IO_SLC_QUEUE_LRW; 1254 1255 rc = setup_ddcb_queue(cd, queue); 1256 if (rc != 0) { 1257 rc = -ENODEV; 1258 goto err_out; 1259 } 1260 1261 init_waitqueue_head(&cd->queue_waitq); 1262 cd->card_thread = kthread_run(genwqe_card_thread, cd, 1263 GENWQE_DEVNAME "%d_thread", 1264 cd->card_idx); 1265 if (IS_ERR(cd->card_thread)) { 1266 rc = PTR_ERR(cd->card_thread); 1267 cd->card_thread = NULL; 1268 goto stop_free_queue; 1269 } 1270 1271 rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS); 1272 if (rc) 1273 goto stop_kthread; 1274 1275 /* 1276 * We must have all wait-queues initialized when we enable the 1277 * interrupts. Otherwise we might crash if we get an early 1278 * irq. 1279 */ 1280 init_waitqueue_head(&cd->health_waitq); 1281 1282 if (genwqe_is_privileged(cd)) { 1283 rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED, 1284 GENWQE_DEVNAME, cd); 1285 } else { 1286 rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED, 1287 GENWQE_DEVNAME, cd); 1288 } 1289 if (rc < 0) { 1290 dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq); 1291 goto stop_irq_cap; 1292 } 1293 1294 cd->card_state = GENWQE_CARD_USED; 1295 return 0; 1296 1297 stop_irq_cap: 1298 genwqe_reset_interrupt_capability(cd); 1299 stop_kthread: 1300 kthread_stop(cd->card_thread); 1301 cd->card_thread = NULL; 1302 stop_free_queue: 1303 free_ddcb_queue(cd, queue); 1304 err_out: 1305 return rc; 1306 } 1307 1308 /** 1309 * queue_wake_up_all() - Handles fatal error case 1310 * 1311 * The PCI device got unusable and we have to stop all pending 1312 * requests as fast as we can. The code after this must purge the 1313 * DDCBs in question and ensure that all mappings are freed. 1314 */ 1315 static int queue_wake_up_all(struct genwqe_dev *cd) 1316 { 1317 unsigned int i; 1318 unsigned long flags; 1319 struct ddcb_queue *queue = &cd->queue; 1320 1321 spin_lock_irqsave(&queue->ddcb_lock, flags); 1322 1323 for (i = 0; i < queue->ddcb_max; i++) 1324 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]); 1325 1326 wake_up_interruptible(&queue->busy_waitq); 1327 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 1328 1329 return 0; 1330 } 1331 1332 /** 1333 * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces 1334 * 1335 * Relies on the pre-condition that there are no users of the card 1336 * device anymore e.g. with open file-descriptors. 1337 * 1338 * This function must be robust enough to be called twice. 1339 */ 1340 int genwqe_finish_queue(struct genwqe_dev *cd) 1341 { 1342 int i, rc = 0, in_flight; 1343 int waitmax = GENWQE_DDCB_SOFTWARE_TIMEOUT; 1344 struct pci_dev *pci_dev = cd->pci_dev; 1345 struct ddcb_queue *queue = &cd->queue; 1346 1347 if (!ddcb_queue_initialized(queue)) 1348 return 0; 1349 1350 /* Do not wipe out the error state. */ 1351 if (cd->card_state == GENWQE_CARD_USED) 1352 cd->card_state = GENWQE_CARD_UNUSED; 1353 1354 /* Wake up all requests in the DDCB queue such that they 1355 should be removed nicely. */ 1356 queue_wake_up_all(cd); 1357 1358 /* We must wait to get rid of the DDCBs in flight */ 1359 for (i = 0; i < waitmax; i++) { 1360 in_flight = genwqe_ddcbs_in_flight(cd); 1361 1362 if (in_flight == 0) 1363 break; 1364 1365 dev_dbg(&pci_dev->dev, 1366 " DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n", 1367 i, waitmax, in_flight); 1368 1369 /* 1370 * Severe severe error situation: The card itself has 1371 * 16 DDCB queues, each queue has e.g. 32 entries, 1372 * each DDBC has a hardware timeout of currently 250 1373 * msec but the PFs have a hardware timeout of 8 sec 1374 * ... so I take something large. 1375 */ 1376 msleep(1000); 1377 } 1378 if (i == waitmax) { 1379 dev_err(&pci_dev->dev, " [%s] err: queue is not empty!!\n", 1380 __func__); 1381 rc = -EIO; 1382 } 1383 return rc; 1384 } 1385 1386 /** 1387 * genwqe_release_service_layer() - Shutdown DDCB queue 1388 * @cd: genwqe device descriptor 1389 * 1390 * This function must be robust enough to be called twice. 1391 */ 1392 int genwqe_release_service_layer(struct genwqe_dev *cd) 1393 { 1394 struct pci_dev *pci_dev = cd->pci_dev; 1395 1396 if (!ddcb_queue_initialized(&cd->queue)) 1397 return 1; 1398 1399 free_irq(pci_dev->irq, cd); 1400 genwqe_reset_interrupt_capability(cd); 1401 1402 if (cd->card_thread != NULL) { 1403 kthread_stop(cd->card_thread); 1404 cd->card_thread = NULL; 1405 } 1406 1407 free_ddcb_queue(cd, &cd->queue); 1408 return 0; 1409 } 1410