1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Linux for s390 qdio support, buffer handling, qdio API and module support. 4 * 5 * Copyright IBM Corp. 2000, 2008 6 * Author(s): Utz Bacher <utz.bacher@de.ibm.com> 7 * Jan Glauber <jang@linux.vnet.ibm.com> 8 * 2.6 cio integration by Cornelia Huck <cornelia.huck@de.ibm.com> 9 */ 10 #include <linux/module.h> 11 #include <linux/init.h> 12 #include <linux/kernel.h> 13 #include <linux/timer.h> 14 #include <linux/delay.h> 15 #include <linux/gfp.h> 16 #include <linux/io.h> 17 #include <linux/atomic.h> 18 #include <asm/debug.h> 19 #include <asm/qdio.h> 20 #include <asm/ipl.h> 21 22 #include "cio.h" 23 #include "css.h" 24 #include "device.h" 25 #include "qdio.h" 26 #include "qdio_debug.h" 27 28 MODULE_AUTHOR("Utz Bacher <utz.bacher@de.ibm.com>,"\ 29 "Jan Glauber <jang@linux.vnet.ibm.com>"); 30 MODULE_DESCRIPTION("QDIO base support"); 31 MODULE_LICENSE("GPL"); 32 33 static inline int do_siga_sync(unsigned long schid, 34 unsigned int out_mask, unsigned int in_mask, 35 unsigned int fc) 36 { 37 register unsigned long __fc asm ("0") = fc; 38 register unsigned long __schid asm ("1") = schid; 39 register unsigned long out asm ("2") = out_mask; 40 register unsigned long in asm ("3") = in_mask; 41 int cc; 42 43 asm volatile( 44 " siga 0\n" 45 " ipm %0\n" 46 " srl %0,28\n" 47 : "=d" (cc) 48 : "d" (__fc), "d" (__schid), "d" (out), "d" (in) : "cc"); 49 return cc; 50 } 51 52 static inline int do_siga_input(unsigned long schid, unsigned int mask, 53 unsigned int fc) 54 { 55 register unsigned long __fc asm ("0") = fc; 56 register unsigned long __schid asm ("1") = schid; 57 register unsigned long __mask asm ("2") = mask; 58 int cc; 59 60 asm volatile( 61 " siga 0\n" 62 " ipm %0\n" 63 " srl %0,28\n" 64 : "=d" (cc) 65 : "d" (__fc), "d" (__schid), "d" (__mask) : "cc"); 66 return cc; 67 } 68 69 /** 70 * do_siga_output - perform SIGA-w/wt function 71 * @schid: subchannel id or in case of QEBSM the subchannel token 72 * @mask: which output queues to process 73 * @bb: busy bit indicator, set only if SIGA-w/wt could not access a buffer 74 * @fc: function code to perform 75 * @aob: asynchronous operation block 76 * 77 * Returns condition code. 78 * Note: For IQDC unicast queues only the highest priority queue is processed. 79 */ 80 static inline int do_siga_output(unsigned long schid, unsigned long mask, 81 unsigned int *bb, unsigned int fc, 82 unsigned long aob) 83 { 84 register unsigned long __fc asm("0") = fc; 85 register unsigned long __schid asm("1") = schid; 86 register unsigned long __mask asm("2") = mask; 87 register unsigned long __aob asm("3") = aob; 88 int cc; 89 90 asm volatile( 91 " siga 0\n" 92 " ipm %0\n" 93 " srl %0,28\n" 94 : "=d" (cc), "+d" (__fc), "+d" (__aob) 95 : "d" (__schid), "d" (__mask) 96 : "cc"); 97 *bb = __fc >> 31; 98 return cc; 99 } 100 101 /** 102 * qdio_do_eqbs - extract buffer states for QEBSM 103 * @q: queue to manipulate 104 * @state: state of the extracted buffers 105 * @start: buffer number to start at 106 * @count: count of buffers to examine 107 * @auto_ack: automatically acknowledge buffers 108 * 109 * Returns the number of successfully extracted equal buffer states. 110 * Stops processing if a state is different from the last buffers state. 111 */ 112 static int qdio_do_eqbs(struct qdio_q *q, unsigned char *state, 113 int start, int count, int auto_ack) 114 { 115 int tmp_count = count, tmp_start = start, nr = q->nr; 116 unsigned int ccq = 0; 117 118 qperf_inc(q, eqbs); 119 120 if (!q->is_input_q) 121 nr += q->irq_ptr->nr_input_qs; 122 again: 123 ccq = do_eqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count, 124 auto_ack); 125 126 switch (ccq) { 127 case 0: 128 case 32: 129 /* all done, or next buffer state different */ 130 return count - tmp_count; 131 case 96: 132 /* not all buffers processed */ 133 qperf_inc(q, eqbs_partial); 134 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "EQBS part:%02x", 135 tmp_count); 136 return count - tmp_count; 137 case 97: 138 /* no buffer processed */ 139 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS again:%2d", ccq); 140 goto again; 141 default: 142 DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq); 143 DBF_ERROR("%4x EQBS ERROR", SCH_NO(q)); 144 DBF_ERROR("%3d%3d%2d", count, tmp_count, nr); 145 q->handler(q->irq_ptr->cdev, QDIO_ERROR_GET_BUF_STATE, q->nr, 146 q->first_to_check, count, q->irq_ptr->int_parm); 147 return 0; 148 } 149 } 150 151 /** 152 * qdio_do_sqbs - set buffer states for QEBSM 153 * @q: queue to manipulate 154 * @state: new state of the buffers 155 * @start: first buffer number to change 156 * @count: how many buffers to change 157 * 158 * Returns the number of successfully changed buffers. 159 * Does retrying until the specified count of buffer states is set or an 160 * error occurs. 161 */ 162 static int qdio_do_sqbs(struct qdio_q *q, unsigned char state, int start, 163 int count) 164 { 165 unsigned int ccq = 0; 166 int tmp_count = count, tmp_start = start; 167 int nr = q->nr; 168 169 if (!count) 170 return 0; 171 qperf_inc(q, sqbs); 172 173 if (!q->is_input_q) 174 nr += q->irq_ptr->nr_input_qs; 175 again: 176 ccq = do_sqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count); 177 178 switch (ccq) { 179 case 0: 180 case 32: 181 /* all done, or active buffer adapter-owned */ 182 WARN_ON_ONCE(tmp_count); 183 return count - tmp_count; 184 case 96: 185 /* not all buffers processed */ 186 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "SQBS again:%2d", ccq); 187 qperf_inc(q, sqbs_partial); 188 goto again; 189 default: 190 DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq); 191 DBF_ERROR("%4x SQBS ERROR", SCH_NO(q)); 192 DBF_ERROR("%3d%3d%2d", count, tmp_count, nr); 193 q->handler(q->irq_ptr->cdev, QDIO_ERROR_SET_BUF_STATE, q->nr, 194 q->first_to_check, count, q->irq_ptr->int_parm); 195 return 0; 196 } 197 } 198 199 /* 200 * Returns number of examined buffers and their common state in *state. 201 * Requested number of buffers-to-examine must be > 0. 202 */ 203 static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr, 204 unsigned char *state, unsigned int count, 205 int auto_ack, int merge_pending) 206 { 207 unsigned char __state = 0; 208 int i = 1; 209 210 if (is_qebsm(q)) 211 return qdio_do_eqbs(q, state, bufnr, count, auto_ack); 212 213 /* get initial state: */ 214 __state = q->slsb.val[bufnr]; 215 216 /* Bail out early if there is no work on the queue: */ 217 if (__state & SLSB_OWNER_CU) 218 goto out; 219 220 if (merge_pending && __state == SLSB_P_OUTPUT_PENDING) 221 __state = SLSB_P_OUTPUT_EMPTY; 222 223 for (; i < count; i++) { 224 bufnr = next_buf(bufnr); 225 226 /* merge PENDING into EMPTY: */ 227 if (merge_pending && 228 q->slsb.val[bufnr] == SLSB_P_OUTPUT_PENDING && 229 __state == SLSB_P_OUTPUT_EMPTY) 230 continue; 231 232 /* stop if next state differs from initial state: */ 233 if (q->slsb.val[bufnr] != __state) 234 break; 235 } 236 237 out: 238 *state = __state; 239 return i; 240 } 241 242 static inline int get_buf_state(struct qdio_q *q, unsigned int bufnr, 243 unsigned char *state, int auto_ack) 244 { 245 return get_buf_states(q, bufnr, state, 1, auto_ack, 0); 246 } 247 248 /* wrap-around safe setting of slsb states, returns number of changed buffers */ 249 static inline int set_buf_states(struct qdio_q *q, int bufnr, 250 unsigned char state, int count) 251 { 252 int i; 253 254 if (is_qebsm(q)) 255 return qdio_do_sqbs(q, state, bufnr, count); 256 257 /* Ensure that all preceding changes to the SBALs are visible: */ 258 mb(); 259 260 for (i = 0; i < count; i++) { 261 WRITE_ONCE(q->slsb.val[bufnr], state); 262 bufnr = next_buf(bufnr); 263 } 264 265 /* Make our SLSB changes visible: */ 266 mb(); 267 268 return count; 269 } 270 271 static inline int set_buf_state(struct qdio_q *q, int bufnr, 272 unsigned char state) 273 { 274 return set_buf_states(q, bufnr, state, 1); 275 } 276 277 /* set slsb states to initial state */ 278 static void qdio_init_buf_states(struct qdio_irq *irq_ptr) 279 { 280 struct qdio_q *q; 281 int i; 282 283 for_each_input_queue(irq_ptr, q, i) 284 set_buf_states(q, 0, SLSB_P_INPUT_NOT_INIT, 285 QDIO_MAX_BUFFERS_PER_Q); 286 for_each_output_queue(irq_ptr, q, i) 287 set_buf_states(q, 0, SLSB_P_OUTPUT_NOT_INIT, 288 QDIO_MAX_BUFFERS_PER_Q); 289 } 290 291 static inline int qdio_siga_sync(struct qdio_q *q, unsigned int output, 292 unsigned int input) 293 { 294 unsigned long schid = *((u32 *) &q->irq_ptr->schid); 295 unsigned int fc = QDIO_SIGA_SYNC; 296 int cc; 297 298 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-s:%1d", q->nr); 299 qperf_inc(q, siga_sync); 300 301 if (is_qebsm(q)) { 302 schid = q->irq_ptr->sch_token; 303 fc |= QDIO_SIGA_QEBSM_FLAG; 304 } 305 306 cc = do_siga_sync(schid, output, input, fc); 307 if (unlikely(cc)) 308 DBF_ERROR("%4x SIGA-S:%2d", SCH_NO(q), cc); 309 return (cc) ? -EIO : 0; 310 } 311 312 static inline int qdio_siga_sync_q(struct qdio_q *q) 313 { 314 if (q->is_input_q) 315 return qdio_siga_sync(q, 0, q->mask); 316 else 317 return qdio_siga_sync(q, q->mask, 0); 318 } 319 320 static int qdio_siga_output(struct qdio_q *q, unsigned int count, 321 unsigned int *busy_bit, unsigned long aob) 322 { 323 unsigned long schid = *((u32 *) &q->irq_ptr->schid); 324 unsigned int fc = QDIO_SIGA_WRITE; 325 u64 start_time = 0; 326 int retries = 0, cc; 327 328 if (queue_type(q) == QDIO_IQDIO_QFMT && !multicast_outbound(q)) { 329 if (count > 1) 330 fc = QDIO_SIGA_WRITEM; 331 else if (aob) 332 fc = QDIO_SIGA_WRITEQ; 333 } 334 335 if (is_qebsm(q)) { 336 schid = q->irq_ptr->sch_token; 337 fc |= QDIO_SIGA_QEBSM_FLAG; 338 } 339 again: 340 cc = do_siga_output(schid, q->mask, busy_bit, fc, aob); 341 342 /* hipersocket busy condition */ 343 if (unlikely(*busy_bit)) { 344 retries++; 345 346 if (!start_time) { 347 start_time = get_tod_clock_fast(); 348 goto again; 349 } 350 if (get_tod_clock_fast() - start_time < QDIO_BUSY_BIT_PATIENCE) 351 goto again; 352 } 353 if (retries) { 354 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, 355 "%4x cc2 BB1:%1d", SCH_NO(q), q->nr); 356 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "count:%u", retries); 357 } 358 return cc; 359 } 360 361 static inline int qdio_siga_input(struct qdio_q *q) 362 { 363 unsigned long schid = *((u32 *) &q->irq_ptr->schid); 364 unsigned int fc = QDIO_SIGA_READ; 365 int cc; 366 367 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-r:%1d", q->nr); 368 qperf_inc(q, siga_read); 369 370 if (is_qebsm(q)) { 371 schid = q->irq_ptr->sch_token; 372 fc |= QDIO_SIGA_QEBSM_FLAG; 373 } 374 375 cc = do_siga_input(schid, q->mask, fc); 376 if (unlikely(cc)) 377 DBF_ERROR("%4x SIGA-R:%2d", SCH_NO(q), cc); 378 return (cc) ? -EIO : 0; 379 } 380 381 #define qdio_siga_sync_out(q) qdio_siga_sync(q, ~0U, 0) 382 #define qdio_siga_sync_all(q) qdio_siga_sync(q, ~0U, ~0U) 383 384 static inline void qdio_sync_queues(struct qdio_q *q) 385 { 386 /* PCI capable outbound queues will also be scanned so sync them too */ 387 if (pci_out_supported(q->irq_ptr)) 388 qdio_siga_sync_all(q); 389 else 390 qdio_siga_sync_q(q); 391 } 392 393 int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr, 394 unsigned char *state) 395 { 396 if (need_siga_sync(q)) 397 qdio_siga_sync_q(q); 398 return get_buf_state(q, bufnr, state, 0); 399 } 400 401 static inline void qdio_stop_polling(struct qdio_q *q) 402 { 403 if (!q->u.in.batch_count) 404 return; 405 406 qperf_inc(q, stop_polling); 407 408 /* show the card that we are not polling anymore */ 409 set_buf_states(q, q->u.in.batch_start, SLSB_P_INPUT_NOT_INIT, 410 q->u.in.batch_count); 411 q->u.in.batch_count = 0; 412 } 413 414 static inline void account_sbals(struct qdio_q *q, unsigned int count) 415 { 416 int pos; 417 418 q->q_stats.nr_sbal_total += count; 419 if (count == QDIO_MAX_BUFFERS_MASK) { 420 q->q_stats.nr_sbals[7]++; 421 return; 422 } 423 pos = ilog2(count); 424 q->q_stats.nr_sbals[pos]++; 425 } 426 427 static void process_buffer_error(struct qdio_q *q, unsigned int start, 428 int count) 429 { 430 q->qdio_error = QDIO_ERROR_SLSB_STATE; 431 432 /* special handling for no target buffer empty */ 433 if (queue_type(q) == QDIO_IQDIO_QFMT && !q->is_input_q && 434 q->sbal[start]->element[15].sflags == 0x10) { 435 qperf_inc(q, target_full); 436 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x", start); 437 return; 438 } 439 440 DBF_ERROR("%4x BUF ERROR", SCH_NO(q)); 441 DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr); 442 DBF_ERROR("FTC:%3d C:%3d", start, count); 443 DBF_ERROR("F14:%2x F15:%2x", 444 q->sbal[start]->element[14].sflags, 445 q->sbal[start]->element[15].sflags); 446 } 447 448 static inline void inbound_handle_work(struct qdio_q *q, unsigned int start, 449 int count, bool auto_ack) 450 { 451 /* ACK the newest SBAL: */ 452 if (!auto_ack) 453 set_buf_state(q, add_buf(start, count - 1), SLSB_P_INPUT_ACK); 454 455 if (!q->u.in.batch_count) 456 q->u.in.batch_start = start; 457 q->u.in.batch_count += count; 458 } 459 460 static int get_inbound_buffer_frontier(struct qdio_q *q, unsigned int start) 461 { 462 unsigned char state = 0; 463 int count; 464 465 q->timestamp = get_tod_clock_fast(); 466 467 /* 468 * Don't check 128 buffers, as otherwise qdio_inbound_q_moved 469 * would return 0. 470 */ 471 count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK); 472 if (!count) 473 return 0; 474 475 /* 476 * No siga sync here, as a PCI or we after a thin interrupt 477 * already sync'ed the queues. 478 */ 479 count = get_buf_states(q, start, &state, count, 1, 0); 480 if (!count) 481 return 0; 482 483 switch (state) { 484 case SLSB_P_INPUT_PRIMED: 485 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim:%1d %02x", q->nr, 486 count); 487 488 inbound_handle_work(q, start, count, is_qebsm(q)); 489 if (atomic_sub_return(count, &q->nr_buf_used) == 0) 490 qperf_inc(q, inbound_queue_full); 491 if (q->irq_ptr->perf_stat_enabled) 492 account_sbals(q, count); 493 return count; 494 case SLSB_P_INPUT_ERROR: 495 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in err:%1d %02x", q->nr, 496 count); 497 498 process_buffer_error(q, start, count); 499 inbound_handle_work(q, start, count, false); 500 if (atomic_sub_return(count, &q->nr_buf_used) == 0) 501 qperf_inc(q, inbound_queue_full); 502 if (q->irq_ptr->perf_stat_enabled) 503 account_sbals_error(q, count); 504 return count; 505 case SLSB_CU_INPUT_EMPTY: 506 if (q->irq_ptr->perf_stat_enabled) 507 q->q_stats.nr_sbal_nop++; 508 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop:%1d %#02x", 509 q->nr, start); 510 return 0; 511 case SLSB_P_INPUT_NOT_INIT: 512 case SLSB_P_INPUT_ACK: 513 /* We should never see this state, throw a WARN: */ 514 default: 515 dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1, 516 "found state %#x at index %u on queue %u\n", 517 state, start, q->nr); 518 return 0; 519 } 520 } 521 522 static int qdio_inbound_q_moved(struct qdio_q *q, unsigned int start) 523 { 524 int count; 525 526 count = get_inbound_buffer_frontier(q, start); 527 528 if (count && !is_thinint_irq(q->irq_ptr) && MACHINE_IS_LPAR) 529 q->u.in.timestamp = get_tod_clock(); 530 531 return count; 532 } 533 534 static inline int qdio_inbound_q_done(struct qdio_q *q, unsigned int start) 535 { 536 unsigned char state = 0; 537 538 if (!atomic_read(&q->nr_buf_used)) 539 return 1; 540 541 if (need_siga_sync(q)) 542 qdio_siga_sync_q(q); 543 get_buf_state(q, start, &state, 0); 544 545 if (state == SLSB_P_INPUT_PRIMED || state == SLSB_P_INPUT_ERROR) 546 /* more work coming */ 547 return 0; 548 549 if (is_thinint_irq(q->irq_ptr)) 550 return 1; 551 552 /* don't poll under z/VM */ 553 if (MACHINE_IS_VM) 554 return 1; 555 556 /* 557 * At this point we know, that inbound first_to_check 558 * has (probably) not moved (see qdio_inbound_processing). 559 */ 560 if (get_tod_clock_fast() > q->u.in.timestamp + QDIO_INPUT_THRESHOLD) { 561 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in done:%02x", start); 562 return 1; 563 } else 564 return 0; 565 } 566 567 static inline void qdio_handle_aobs(struct qdio_q *q, int start, int count) 568 { 569 unsigned char state = 0; 570 int j, b = start; 571 572 for (j = 0; j < count; ++j) { 573 get_buf_state(q, b, &state, 0); 574 if (state == SLSB_P_OUTPUT_PENDING) { 575 struct qaob *aob = q->u.out.aobs[b]; 576 if (aob == NULL) 577 continue; 578 579 q->u.out.sbal_state[b].flags |= 580 QDIO_OUTBUF_STATE_FLAG_PENDING; 581 q->u.out.aobs[b] = NULL; 582 } 583 b = next_buf(b); 584 } 585 } 586 587 static inline unsigned long qdio_aob_for_buffer(struct qdio_output_q *q, 588 int bufnr) 589 { 590 unsigned long phys_aob = 0; 591 592 if (!q->aobs[bufnr]) { 593 struct qaob *aob = qdio_allocate_aob(); 594 q->aobs[bufnr] = aob; 595 } 596 if (q->aobs[bufnr]) { 597 q->aobs[bufnr]->user1 = (u64) q->sbal_state[bufnr].user; 598 phys_aob = virt_to_phys(q->aobs[bufnr]); 599 WARN_ON_ONCE(phys_aob & 0xFF); 600 } 601 602 q->sbal_state[bufnr].flags = 0; 603 return phys_aob; 604 } 605 606 static void qdio_kick_handler(struct qdio_q *q, unsigned int start, 607 unsigned int count) 608 { 609 if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)) 610 return; 611 612 if (q->is_input_q) { 613 qperf_inc(q, inbound_handler); 614 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "kih s:%02x c:%02x", start, count); 615 } else { 616 qperf_inc(q, outbound_handler); 617 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "koh: s:%02x c:%02x", 618 start, count); 619 } 620 621 q->handler(q->irq_ptr->cdev, q->qdio_error, q->nr, start, count, 622 q->irq_ptr->int_parm); 623 624 /* for the next time */ 625 q->qdio_error = 0; 626 } 627 628 static inline int qdio_tasklet_schedule(struct qdio_q *q) 629 { 630 if (likely(q->irq_ptr->state == QDIO_IRQ_STATE_ACTIVE)) { 631 tasklet_schedule(&q->tasklet); 632 return 0; 633 } 634 return -EPERM; 635 } 636 637 static void __qdio_inbound_processing(struct qdio_q *q) 638 { 639 unsigned int start = q->first_to_check; 640 int count; 641 642 qperf_inc(q, tasklet_inbound); 643 644 count = qdio_inbound_q_moved(q, start); 645 if (count == 0) 646 return; 647 648 qdio_kick_handler(q, start, count); 649 start = add_buf(start, count); 650 q->first_to_check = start; 651 652 if (!qdio_inbound_q_done(q, start)) { 653 /* means poll time is not yet over */ 654 qperf_inc(q, tasklet_inbound_resched); 655 if (!qdio_tasklet_schedule(q)) 656 return; 657 } 658 659 qdio_stop_polling(q); 660 /* 661 * We need to check again to not lose initiative after 662 * resetting the ACK state. 663 */ 664 if (!qdio_inbound_q_done(q, start)) { 665 qperf_inc(q, tasklet_inbound_resched2); 666 qdio_tasklet_schedule(q); 667 } 668 } 669 670 void qdio_inbound_processing(unsigned long data) 671 { 672 struct qdio_q *q = (struct qdio_q *)data; 673 __qdio_inbound_processing(q); 674 } 675 676 static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start) 677 { 678 unsigned char state = 0; 679 int count; 680 681 q->timestamp = get_tod_clock_fast(); 682 683 if (need_siga_sync(q)) 684 if (((queue_type(q) != QDIO_IQDIO_QFMT) && 685 !pci_out_supported(q->irq_ptr)) || 686 (queue_type(q) == QDIO_IQDIO_QFMT && 687 multicast_outbound(q))) 688 qdio_siga_sync_q(q); 689 690 count = atomic_read(&q->nr_buf_used); 691 if (!count) 692 return 0; 693 694 count = get_buf_states(q, start, &state, count, 0, q->u.out.use_cq); 695 if (!count) 696 return 0; 697 698 switch (state) { 699 case SLSB_P_OUTPUT_EMPTY: 700 case SLSB_P_OUTPUT_PENDING: 701 /* the adapter got it */ 702 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, 703 "out empty:%1d %02x", q->nr, count); 704 705 atomic_sub(count, &q->nr_buf_used); 706 if (q->irq_ptr->perf_stat_enabled) 707 account_sbals(q, count); 708 return count; 709 case SLSB_P_OUTPUT_ERROR: 710 process_buffer_error(q, start, count); 711 atomic_sub(count, &q->nr_buf_used); 712 if (q->irq_ptr->perf_stat_enabled) 713 account_sbals_error(q, count); 714 return count; 715 case SLSB_CU_OUTPUT_PRIMED: 716 /* the adapter has not fetched the output yet */ 717 if (q->irq_ptr->perf_stat_enabled) 718 q->q_stats.nr_sbal_nop++; 719 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d", 720 q->nr); 721 return 0; 722 case SLSB_P_OUTPUT_HALTED: 723 return 0; 724 case SLSB_P_OUTPUT_NOT_INIT: 725 /* We should never see this state, throw a WARN: */ 726 default: 727 dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1, 728 "found state %#x at index %u on queue %u\n", 729 state, start, q->nr); 730 return 0; 731 } 732 } 733 734 /* all buffers processed? */ 735 static inline int qdio_outbound_q_done(struct qdio_q *q) 736 { 737 return atomic_read(&q->nr_buf_used) == 0; 738 } 739 740 static inline int qdio_outbound_q_moved(struct qdio_q *q, unsigned int start) 741 { 742 int count; 743 744 count = get_outbound_buffer_frontier(q, start); 745 746 if (count) { 747 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out moved:%1d", q->nr); 748 if (q->u.out.use_cq) 749 qdio_handle_aobs(q, start, count); 750 } 751 752 return count; 753 } 754 755 static int qdio_kick_outbound_q(struct qdio_q *q, unsigned int count, 756 unsigned long aob) 757 { 758 int retries = 0, cc; 759 unsigned int busy_bit; 760 761 if (!need_siga_out(q)) 762 return 0; 763 764 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w:%1d", q->nr); 765 retry: 766 qperf_inc(q, siga_write); 767 768 cc = qdio_siga_output(q, count, &busy_bit, aob); 769 switch (cc) { 770 case 0: 771 break; 772 case 2: 773 if (busy_bit) { 774 while (++retries < QDIO_BUSY_BIT_RETRIES) { 775 mdelay(QDIO_BUSY_BIT_RETRY_DELAY); 776 goto retry; 777 } 778 DBF_ERROR("%4x cc2 BBC:%1d", SCH_NO(q), q->nr); 779 cc = -EBUSY; 780 } else { 781 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w cc2:%1d", q->nr); 782 cc = -ENOBUFS; 783 } 784 break; 785 case 1: 786 case 3: 787 DBF_ERROR("%4x SIGA-W:%1d", SCH_NO(q), cc); 788 cc = -EIO; 789 break; 790 } 791 if (retries) { 792 DBF_ERROR("%4x cc2 BB2:%1d", SCH_NO(q), q->nr); 793 DBF_ERROR("count:%u", retries); 794 } 795 return cc; 796 } 797 798 static void __qdio_outbound_processing(struct qdio_q *q) 799 { 800 unsigned int start = q->first_to_check; 801 int count; 802 803 qperf_inc(q, tasklet_outbound); 804 WARN_ON_ONCE(atomic_read(&q->nr_buf_used) < 0); 805 806 count = qdio_outbound_q_moved(q, start); 807 if (count) { 808 q->first_to_check = add_buf(start, count); 809 qdio_kick_handler(q, start, count); 810 } 811 812 if (queue_type(q) == QDIO_ZFCP_QFMT && !pci_out_supported(q->irq_ptr) && 813 !qdio_outbound_q_done(q)) 814 goto sched; 815 816 if (q->u.out.pci_out_enabled) 817 return; 818 819 /* 820 * Now we know that queue type is either qeth without pci enabled 821 * or HiperSockets. Make sure buffer switch from PRIMED to EMPTY 822 * is noticed and outbound_handler is called after some time. 823 */ 824 if (qdio_outbound_q_done(q)) 825 del_timer_sync(&q->u.out.timer); 826 else 827 if (!timer_pending(&q->u.out.timer) && 828 likely(q->irq_ptr->state == QDIO_IRQ_STATE_ACTIVE)) 829 mod_timer(&q->u.out.timer, jiffies + 10 * HZ); 830 return; 831 832 sched: 833 qdio_tasklet_schedule(q); 834 } 835 836 /* outbound tasklet */ 837 void qdio_outbound_processing(unsigned long data) 838 { 839 struct qdio_q *q = (struct qdio_q *)data; 840 __qdio_outbound_processing(q); 841 } 842 843 void qdio_outbound_timer(struct timer_list *t) 844 { 845 struct qdio_q *q = from_timer(q, t, u.out.timer); 846 847 qdio_tasklet_schedule(q); 848 } 849 850 static inline void qdio_check_outbound_pci_queues(struct qdio_irq *irq) 851 { 852 struct qdio_q *out; 853 int i; 854 855 if (!pci_out_supported(irq) || !irq->scan_threshold) 856 return; 857 858 for_each_output_queue(irq, out, i) 859 if (!qdio_outbound_q_done(out)) 860 qdio_tasklet_schedule(out); 861 } 862 863 void tiqdio_inbound_processing(unsigned long data) 864 { 865 struct qdio_q *q = (struct qdio_q *)data; 866 867 if (need_siga_sync(q) && need_siga_sync_after_ai(q)) 868 qdio_sync_queues(q); 869 870 /* The interrupt could be caused by a PCI request: */ 871 qdio_check_outbound_pci_queues(q->irq_ptr); 872 873 __qdio_inbound_processing(q); 874 } 875 876 static inline void qdio_set_state(struct qdio_irq *irq_ptr, 877 enum qdio_irq_states state) 878 { 879 DBF_DEV_EVENT(DBF_INFO, irq_ptr, "newstate: %1d", state); 880 881 irq_ptr->state = state; 882 mb(); 883 } 884 885 static void qdio_irq_check_sense(struct qdio_irq *irq_ptr, struct irb *irb) 886 { 887 if (irb->esw.esw0.erw.cons) { 888 DBF_ERROR("%4x sense:", irq_ptr->schid.sch_no); 889 DBF_ERROR_HEX(irb, 64); 890 DBF_ERROR_HEX(irb->ecw, 64); 891 } 892 } 893 894 /* PCI interrupt handler */ 895 static void qdio_int_handler_pci(struct qdio_irq *irq_ptr) 896 { 897 int i; 898 struct qdio_q *q; 899 900 if (unlikely(irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)) 901 return; 902 903 if (irq_ptr->irq_poll) { 904 if (!test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state)) 905 irq_ptr->irq_poll(irq_ptr->cdev, irq_ptr->int_parm); 906 else 907 QDIO_PERF_STAT_INC(irq_ptr, int_discarded); 908 } else { 909 for_each_input_queue(irq_ptr, q, i) 910 tasklet_schedule(&q->tasklet); 911 } 912 913 if (!pci_out_supported(irq_ptr) || !irq_ptr->scan_threshold) 914 return; 915 916 for_each_output_queue(irq_ptr, q, i) { 917 if (qdio_outbound_q_done(q)) 918 continue; 919 if (need_siga_sync(q) && need_siga_sync_out_after_pci(q)) 920 qdio_siga_sync_q(q); 921 qdio_tasklet_schedule(q); 922 } 923 } 924 925 static void qdio_handle_activate_check(struct qdio_irq *irq_ptr, 926 unsigned long intparm, int cstat, 927 int dstat) 928 { 929 struct qdio_q *q; 930 931 DBF_ERROR("%4x ACT CHECK", irq_ptr->schid.sch_no); 932 DBF_ERROR("intp :%lx", intparm); 933 DBF_ERROR("ds: %2x cs:%2x", dstat, cstat); 934 935 if (irq_ptr->nr_input_qs) { 936 q = irq_ptr->input_qs[0]; 937 } else if (irq_ptr->nr_output_qs) { 938 q = irq_ptr->output_qs[0]; 939 } else { 940 dump_stack(); 941 goto no_handler; 942 } 943 944 q->handler(q->irq_ptr->cdev, QDIO_ERROR_ACTIVATE, 945 q->nr, q->first_to_check, 0, irq_ptr->int_parm); 946 no_handler: 947 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED); 948 /* 949 * In case of z/VM LGR (Live Guest Migration) QDIO recovery will happen. 950 * Therefore we call the LGR detection function here. 951 */ 952 lgr_info_log(); 953 } 954 955 static void qdio_establish_handle_irq(struct qdio_irq *irq_ptr, int cstat, 956 int dstat) 957 { 958 DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qest irq"); 959 960 if (cstat) 961 goto error; 962 if (dstat & ~(DEV_STAT_DEV_END | DEV_STAT_CHN_END)) 963 goto error; 964 if (!(dstat & DEV_STAT_DEV_END)) 965 goto error; 966 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ESTABLISHED); 967 return; 968 969 error: 970 DBF_ERROR("%4x EQ:error", irq_ptr->schid.sch_no); 971 DBF_ERROR("ds: %2x cs:%2x", dstat, cstat); 972 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR); 973 } 974 975 /* qdio interrupt handler */ 976 void qdio_int_handler(struct ccw_device *cdev, unsigned long intparm, 977 struct irb *irb) 978 { 979 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 980 struct subchannel_id schid; 981 int cstat, dstat; 982 983 if (!intparm || !irq_ptr) { 984 ccw_device_get_schid(cdev, &schid); 985 DBF_ERROR("qint:%4x", schid.sch_no); 986 return; 987 } 988 989 if (irq_ptr->perf_stat_enabled) 990 irq_ptr->perf_stat.qdio_int++; 991 992 if (IS_ERR(irb)) { 993 DBF_ERROR("%4x IO error", irq_ptr->schid.sch_no); 994 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR); 995 wake_up(&cdev->private->wait_q); 996 return; 997 } 998 qdio_irq_check_sense(irq_ptr, irb); 999 cstat = irb->scsw.cmd.cstat; 1000 dstat = irb->scsw.cmd.dstat; 1001 1002 switch (irq_ptr->state) { 1003 case QDIO_IRQ_STATE_INACTIVE: 1004 qdio_establish_handle_irq(irq_ptr, cstat, dstat); 1005 break; 1006 case QDIO_IRQ_STATE_CLEANUP: 1007 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); 1008 break; 1009 case QDIO_IRQ_STATE_ESTABLISHED: 1010 case QDIO_IRQ_STATE_ACTIVE: 1011 if (cstat & SCHN_STAT_PCI) { 1012 qdio_int_handler_pci(irq_ptr); 1013 return; 1014 } 1015 if (cstat || dstat) 1016 qdio_handle_activate_check(irq_ptr, intparm, cstat, 1017 dstat); 1018 break; 1019 case QDIO_IRQ_STATE_STOPPED: 1020 break; 1021 default: 1022 WARN_ON_ONCE(1); 1023 } 1024 wake_up(&cdev->private->wait_q); 1025 } 1026 1027 /** 1028 * qdio_get_ssqd_desc - get qdio subchannel description 1029 * @cdev: ccw device to get description for 1030 * @data: where to store the ssqd 1031 * 1032 * Returns 0 or an error code. The results of the chsc are stored in the 1033 * specified structure. 1034 */ 1035 int qdio_get_ssqd_desc(struct ccw_device *cdev, 1036 struct qdio_ssqd_desc *data) 1037 { 1038 struct subchannel_id schid; 1039 1040 if (!cdev || !cdev->private) 1041 return -EINVAL; 1042 1043 ccw_device_get_schid(cdev, &schid); 1044 DBF_EVENT("get ssqd:%4x", schid.sch_no); 1045 return qdio_setup_get_ssqd(NULL, &schid, data); 1046 } 1047 EXPORT_SYMBOL_GPL(qdio_get_ssqd_desc); 1048 1049 static void qdio_shutdown_queues(struct qdio_irq *irq_ptr) 1050 { 1051 struct qdio_q *q; 1052 int i; 1053 1054 for_each_input_queue(irq_ptr, q, i) 1055 tasklet_kill(&q->tasklet); 1056 1057 for_each_output_queue(irq_ptr, q, i) { 1058 del_timer_sync(&q->u.out.timer); 1059 tasklet_kill(&q->tasklet); 1060 } 1061 } 1062 1063 /** 1064 * qdio_shutdown - shut down a qdio subchannel 1065 * @cdev: associated ccw device 1066 * @how: use halt or clear to shutdown 1067 */ 1068 int qdio_shutdown(struct ccw_device *cdev, int how) 1069 { 1070 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1071 struct subchannel_id schid; 1072 int rc; 1073 1074 if (!irq_ptr) 1075 return -ENODEV; 1076 1077 WARN_ON_ONCE(irqs_disabled()); 1078 ccw_device_get_schid(cdev, &schid); 1079 DBF_EVENT("qshutdown:%4x", schid.sch_no); 1080 1081 mutex_lock(&irq_ptr->setup_mutex); 1082 /* 1083 * Subchannel was already shot down. We cannot prevent being called 1084 * twice since cio may trigger a shutdown asynchronously. 1085 */ 1086 if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) { 1087 mutex_unlock(&irq_ptr->setup_mutex); 1088 return 0; 1089 } 1090 1091 /* 1092 * Indicate that the device is going down. Scheduling the queue 1093 * tasklets is forbidden from here on. 1094 */ 1095 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED); 1096 1097 tiqdio_remove_device(irq_ptr); 1098 qdio_shutdown_queues(irq_ptr); 1099 qdio_shutdown_debug_entries(irq_ptr); 1100 1101 /* cleanup subchannel */ 1102 spin_lock_irq(get_ccwdev_lock(cdev)); 1103 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_CLEANUP); 1104 if (how & QDIO_FLAG_CLEANUP_USING_CLEAR) 1105 rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP); 1106 else 1107 /* default behaviour is halt */ 1108 rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP); 1109 spin_unlock_irq(get_ccwdev_lock(cdev)); 1110 if (rc) { 1111 DBF_ERROR("%4x SHUTD ERR", irq_ptr->schid.sch_no); 1112 DBF_ERROR("rc:%4d", rc); 1113 goto no_cleanup; 1114 } 1115 1116 wait_event_interruptible_timeout(cdev->private->wait_q, 1117 irq_ptr->state == QDIO_IRQ_STATE_INACTIVE || 1118 irq_ptr->state == QDIO_IRQ_STATE_ERR, 1119 10 * HZ); 1120 1121 no_cleanup: 1122 qdio_shutdown_thinint(irq_ptr); 1123 qdio_shutdown_irq(irq_ptr); 1124 1125 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); 1126 mutex_unlock(&irq_ptr->setup_mutex); 1127 if (rc) 1128 return rc; 1129 return 0; 1130 } 1131 EXPORT_SYMBOL_GPL(qdio_shutdown); 1132 1133 /** 1134 * qdio_free - free data structures for a qdio subchannel 1135 * @cdev: associated ccw device 1136 */ 1137 int qdio_free(struct ccw_device *cdev) 1138 { 1139 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1140 struct subchannel_id schid; 1141 1142 if (!irq_ptr) 1143 return -ENODEV; 1144 1145 ccw_device_get_schid(cdev, &schid); 1146 DBF_EVENT("qfree:%4x", schid.sch_no); 1147 DBF_DEV_EVENT(DBF_ERR, irq_ptr, "dbf abandoned"); 1148 mutex_lock(&irq_ptr->setup_mutex); 1149 1150 irq_ptr->debug_area = NULL; 1151 cdev->private->qdio_data = NULL; 1152 mutex_unlock(&irq_ptr->setup_mutex); 1153 1154 qdio_free_async_data(irq_ptr); 1155 qdio_free_queues(irq_ptr); 1156 free_page((unsigned long) irq_ptr->qdr); 1157 free_page(irq_ptr->chsc_page); 1158 free_page((unsigned long) irq_ptr); 1159 return 0; 1160 } 1161 EXPORT_SYMBOL_GPL(qdio_free); 1162 1163 /** 1164 * qdio_allocate - allocate qdio queues and associated data 1165 * @cdev: associated ccw device 1166 * @no_input_qs: allocate this number of Input Queues 1167 * @no_output_qs: allocate this number of Output Queues 1168 */ 1169 int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs, 1170 unsigned int no_output_qs) 1171 { 1172 struct subchannel_id schid; 1173 struct qdio_irq *irq_ptr; 1174 int rc = -ENOMEM; 1175 1176 ccw_device_get_schid(cdev, &schid); 1177 DBF_EVENT("qallocate:%4x", schid.sch_no); 1178 1179 if (no_input_qs > QDIO_MAX_QUEUES_PER_IRQ || 1180 no_output_qs > QDIO_MAX_QUEUES_PER_IRQ) 1181 return -EINVAL; 1182 1183 /* irq_ptr must be in GFP_DMA since it contains ccw1.cda */ 1184 irq_ptr = (void *) get_zeroed_page(GFP_KERNEL | GFP_DMA); 1185 if (!irq_ptr) 1186 return -ENOMEM; 1187 1188 irq_ptr->cdev = cdev; 1189 mutex_init(&irq_ptr->setup_mutex); 1190 if (qdio_allocate_dbf(irq_ptr)) 1191 goto err_dbf; 1192 1193 DBF_DEV_EVENT(DBF_ERR, irq_ptr, "alloc niq:%1u noq:%1u", no_input_qs, 1194 no_output_qs); 1195 1196 /* 1197 * Allocate a page for the chsc calls in qdio_establish. 1198 * Must be pre-allocated since a zfcp recovery will call 1199 * qdio_establish. In case of low memory and swap on a zfcp disk 1200 * we may not be able to allocate memory otherwise. 1201 */ 1202 irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL); 1203 if (!irq_ptr->chsc_page) 1204 goto err_chsc; 1205 1206 /* qdr is used in ccw1.cda which is u32 */ 1207 irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL | GFP_DMA); 1208 if (!irq_ptr->qdr) 1209 goto err_qdr; 1210 1211 rc = qdio_allocate_qs(irq_ptr, no_input_qs, no_output_qs); 1212 if (rc) 1213 goto err_queues; 1214 1215 INIT_LIST_HEAD(&irq_ptr->entry); 1216 cdev->private->qdio_data = irq_ptr; 1217 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); 1218 return 0; 1219 1220 err_queues: 1221 free_page((unsigned long) irq_ptr->qdr); 1222 err_qdr: 1223 free_page(irq_ptr->chsc_page); 1224 err_chsc: 1225 err_dbf: 1226 free_page((unsigned long) irq_ptr); 1227 return rc; 1228 } 1229 EXPORT_SYMBOL_GPL(qdio_allocate); 1230 1231 static void qdio_detect_hsicq(struct qdio_irq *irq_ptr) 1232 { 1233 struct qdio_q *q = irq_ptr->input_qs[0]; 1234 int i, use_cq = 0; 1235 1236 if (irq_ptr->nr_input_qs > 1 && queue_type(q) == QDIO_IQDIO_QFMT) 1237 use_cq = 1; 1238 1239 for_each_output_queue(irq_ptr, q, i) { 1240 if (use_cq) { 1241 if (multicast_outbound(q)) 1242 continue; 1243 if (qdio_enable_async_operation(&q->u.out) < 0) { 1244 use_cq = 0; 1245 continue; 1246 } 1247 } else 1248 qdio_disable_async_operation(&q->u.out); 1249 } 1250 DBF_EVENT("use_cq:%d", use_cq); 1251 } 1252 1253 static void qdio_trace_init_data(struct qdio_irq *irq, 1254 struct qdio_initialize *data) 1255 { 1256 DBF_DEV_EVENT(DBF_ERR, irq, "qfmt:%1u", data->q_format); 1257 DBF_DEV_HEX(irq, data->adapter_name, 8, DBF_ERR); 1258 DBF_DEV_EVENT(DBF_ERR, irq, "qpff%4x", data->qib_param_field_format); 1259 DBF_DEV_HEX(irq, &data->qib_param_field, sizeof(void *), DBF_ERR); 1260 DBF_DEV_HEX(irq, &data->input_slib_elements, sizeof(void *), DBF_ERR); 1261 DBF_DEV_HEX(irq, &data->output_slib_elements, sizeof(void *), DBF_ERR); 1262 DBF_DEV_EVENT(DBF_ERR, irq, "niq:%1u noq:%1u", data->no_input_qs, 1263 data->no_output_qs); 1264 DBF_DEV_HEX(irq, &data->input_handler, sizeof(void *), DBF_ERR); 1265 DBF_DEV_HEX(irq, &data->output_handler, sizeof(void *), DBF_ERR); 1266 DBF_DEV_HEX(irq, &data->int_parm, sizeof(long), DBF_ERR); 1267 DBF_DEV_HEX(irq, &data->input_sbal_addr_array, sizeof(void *), DBF_ERR); 1268 DBF_DEV_HEX(irq, &data->output_sbal_addr_array, sizeof(void *), 1269 DBF_ERR); 1270 } 1271 1272 /** 1273 * qdio_establish - establish queues on a qdio subchannel 1274 * @cdev: associated ccw device 1275 * @init_data: initialization data 1276 */ 1277 int qdio_establish(struct ccw_device *cdev, 1278 struct qdio_initialize *init_data) 1279 { 1280 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1281 struct subchannel_id schid; 1282 int rc; 1283 1284 ccw_device_get_schid(cdev, &schid); 1285 DBF_EVENT("qestablish:%4x", schid.sch_no); 1286 1287 if (!irq_ptr) 1288 return -ENODEV; 1289 1290 if (init_data->no_input_qs > irq_ptr->max_input_qs || 1291 init_data->no_output_qs > irq_ptr->max_output_qs) 1292 return -EINVAL; 1293 1294 if ((init_data->no_input_qs && !init_data->input_handler) || 1295 (init_data->no_output_qs && !init_data->output_handler)) 1296 return -EINVAL; 1297 1298 if (!init_data->input_sbal_addr_array || 1299 !init_data->output_sbal_addr_array) 1300 return -EINVAL; 1301 1302 mutex_lock(&irq_ptr->setup_mutex); 1303 qdio_trace_init_data(irq_ptr, init_data); 1304 qdio_setup_irq(irq_ptr, init_data); 1305 1306 rc = qdio_establish_thinint(irq_ptr); 1307 if (rc) { 1308 qdio_shutdown_irq(irq_ptr); 1309 mutex_unlock(&irq_ptr->setup_mutex); 1310 return rc; 1311 } 1312 1313 /* establish q */ 1314 irq_ptr->ccw.cmd_code = irq_ptr->equeue.cmd; 1315 irq_ptr->ccw.flags = CCW_FLAG_SLI; 1316 irq_ptr->ccw.count = irq_ptr->equeue.count; 1317 irq_ptr->ccw.cda = (u32)((addr_t)irq_ptr->qdr); 1318 1319 spin_lock_irq(get_ccwdev_lock(cdev)); 1320 ccw_device_set_options_mask(cdev, 0); 1321 1322 rc = ccw_device_start(cdev, &irq_ptr->ccw, QDIO_DOING_ESTABLISH, 0, 0); 1323 spin_unlock_irq(get_ccwdev_lock(cdev)); 1324 if (rc) { 1325 DBF_ERROR("%4x est IO ERR", irq_ptr->schid.sch_no); 1326 DBF_ERROR("rc:%4x", rc); 1327 qdio_shutdown_thinint(irq_ptr); 1328 qdio_shutdown_irq(irq_ptr); 1329 mutex_unlock(&irq_ptr->setup_mutex); 1330 return rc; 1331 } 1332 1333 wait_event_interruptible_timeout(cdev->private->wait_q, 1334 irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED || 1335 irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ); 1336 1337 if (irq_ptr->state != QDIO_IRQ_STATE_ESTABLISHED) { 1338 mutex_unlock(&irq_ptr->setup_mutex); 1339 qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR); 1340 return -EIO; 1341 } 1342 1343 qdio_setup_ssqd_info(irq_ptr); 1344 1345 qdio_detect_hsicq(irq_ptr); 1346 1347 /* qebsm is now setup if available, initialize buffer states */ 1348 qdio_init_buf_states(irq_ptr); 1349 1350 mutex_unlock(&irq_ptr->setup_mutex); 1351 qdio_print_subchannel_info(irq_ptr); 1352 qdio_setup_debug_entries(irq_ptr); 1353 return 0; 1354 } 1355 EXPORT_SYMBOL_GPL(qdio_establish); 1356 1357 /** 1358 * qdio_activate - activate queues on a qdio subchannel 1359 * @cdev: associated cdev 1360 */ 1361 int qdio_activate(struct ccw_device *cdev) 1362 { 1363 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1364 struct subchannel_id schid; 1365 int rc; 1366 1367 ccw_device_get_schid(cdev, &schid); 1368 DBF_EVENT("qactivate:%4x", schid.sch_no); 1369 1370 if (!irq_ptr) 1371 return -ENODEV; 1372 1373 mutex_lock(&irq_ptr->setup_mutex); 1374 if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) { 1375 rc = -EBUSY; 1376 goto out; 1377 } 1378 1379 irq_ptr->ccw.cmd_code = irq_ptr->aqueue.cmd; 1380 irq_ptr->ccw.flags = CCW_FLAG_SLI; 1381 irq_ptr->ccw.count = irq_ptr->aqueue.count; 1382 irq_ptr->ccw.cda = 0; 1383 1384 spin_lock_irq(get_ccwdev_lock(cdev)); 1385 ccw_device_set_options(cdev, CCWDEV_REPORT_ALL); 1386 1387 rc = ccw_device_start(cdev, &irq_ptr->ccw, QDIO_DOING_ACTIVATE, 1388 0, DOIO_DENY_PREFETCH); 1389 spin_unlock_irq(get_ccwdev_lock(cdev)); 1390 if (rc) { 1391 DBF_ERROR("%4x act IO ERR", irq_ptr->schid.sch_no); 1392 DBF_ERROR("rc:%4x", rc); 1393 goto out; 1394 } 1395 1396 if (is_thinint_irq(irq_ptr)) 1397 tiqdio_add_device(irq_ptr); 1398 1399 /* wait for subchannel to become active */ 1400 msleep(5); 1401 1402 switch (irq_ptr->state) { 1403 case QDIO_IRQ_STATE_STOPPED: 1404 case QDIO_IRQ_STATE_ERR: 1405 rc = -EIO; 1406 break; 1407 default: 1408 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ACTIVE); 1409 rc = 0; 1410 } 1411 out: 1412 mutex_unlock(&irq_ptr->setup_mutex); 1413 return rc; 1414 } 1415 EXPORT_SYMBOL_GPL(qdio_activate); 1416 1417 /** 1418 * handle_inbound - reset processed input buffers 1419 * @q: queue containing the buffers 1420 * @callflags: flags 1421 * @bufnr: first buffer to process 1422 * @count: how many buffers are emptied 1423 */ 1424 static int handle_inbound(struct qdio_q *q, unsigned int callflags, 1425 int bufnr, int count) 1426 { 1427 int overlap; 1428 1429 qperf_inc(q, inbound_call); 1430 1431 /* If any processed SBALs are returned to HW, adjust our tracking: */ 1432 overlap = min_t(int, count - sub_buf(q->u.in.batch_start, bufnr), 1433 q->u.in.batch_count); 1434 if (overlap > 0) { 1435 q->u.in.batch_start = add_buf(q->u.in.batch_start, overlap); 1436 q->u.in.batch_count -= overlap; 1437 } 1438 1439 count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count); 1440 atomic_add(count, &q->nr_buf_used); 1441 1442 if (need_siga_in(q)) 1443 return qdio_siga_input(q); 1444 1445 return 0; 1446 } 1447 1448 /** 1449 * handle_outbound - process filled outbound buffers 1450 * @q: queue containing the buffers 1451 * @callflags: flags 1452 * @bufnr: first buffer to process 1453 * @count: how many buffers are filled 1454 */ 1455 static int handle_outbound(struct qdio_q *q, unsigned int callflags, 1456 unsigned int bufnr, unsigned int count) 1457 { 1458 const unsigned int scan_threshold = q->irq_ptr->scan_threshold; 1459 unsigned char state = 0; 1460 int used, rc = 0; 1461 1462 qperf_inc(q, outbound_call); 1463 1464 count = set_buf_states(q, bufnr, SLSB_CU_OUTPUT_PRIMED, count); 1465 used = atomic_add_return(count, &q->nr_buf_used); 1466 1467 if (used == QDIO_MAX_BUFFERS_PER_Q) 1468 qperf_inc(q, outbound_queue_full); 1469 1470 if (callflags & QDIO_FLAG_PCI_OUT) { 1471 q->u.out.pci_out_enabled = 1; 1472 qperf_inc(q, pci_request_int); 1473 } else 1474 q->u.out.pci_out_enabled = 0; 1475 1476 if (queue_type(q) == QDIO_IQDIO_QFMT) { 1477 unsigned long phys_aob = 0; 1478 1479 if (q->u.out.use_cq && count == 1) 1480 phys_aob = qdio_aob_for_buffer(&q->u.out, bufnr); 1481 1482 rc = qdio_kick_outbound_q(q, count, phys_aob); 1483 } else if (need_siga_sync(q)) { 1484 rc = qdio_siga_sync_q(q); 1485 } else if (count < QDIO_MAX_BUFFERS_PER_Q && 1486 get_buf_state(q, prev_buf(bufnr), &state, 0) > 0 && 1487 state == SLSB_CU_OUTPUT_PRIMED) { 1488 /* The previous buffer is not processed yet, tack on. */ 1489 qperf_inc(q, fast_requeue); 1490 } else { 1491 rc = qdio_kick_outbound_q(q, count, 0); 1492 } 1493 1494 /* Let drivers implement their own completion scanning: */ 1495 if (!scan_threshold) 1496 return rc; 1497 1498 /* in case of SIGA errors we must process the error immediately */ 1499 if (used >= scan_threshold || rc) 1500 qdio_tasklet_schedule(q); 1501 else 1502 /* free the SBALs in case of no further traffic */ 1503 if (!timer_pending(&q->u.out.timer) && 1504 likely(q->irq_ptr->state == QDIO_IRQ_STATE_ACTIVE)) 1505 mod_timer(&q->u.out.timer, jiffies + HZ); 1506 return rc; 1507 } 1508 1509 /** 1510 * do_QDIO - process input or output buffers 1511 * @cdev: associated ccw_device for the qdio subchannel 1512 * @callflags: input or output and special flags from the program 1513 * @q_nr: queue number 1514 * @bufnr: buffer number 1515 * @count: how many buffers to process 1516 */ 1517 int do_QDIO(struct ccw_device *cdev, unsigned int callflags, 1518 int q_nr, unsigned int bufnr, unsigned int count) 1519 { 1520 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1521 1522 if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q) 1523 return -EINVAL; 1524 1525 if (!irq_ptr) 1526 return -ENODEV; 1527 1528 DBF_DEV_EVENT(DBF_INFO, irq_ptr, 1529 "do%02x b:%02x c:%02x", callflags, bufnr, count); 1530 1531 if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE) 1532 return -EIO; 1533 if (!count) 1534 return 0; 1535 if (callflags & QDIO_FLAG_SYNC_INPUT) 1536 return handle_inbound(irq_ptr->input_qs[q_nr], 1537 callflags, bufnr, count); 1538 else if (callflags & QDIO_FLAG_SYNC_OUTPUT) 1539 return handle_outbound(irq_ptr->output_qs[q_nr], 1540 callflags, bufnr, count); 1541 return -EINVAL; 1542 } 1543 EXPORT_SYMBOL_GPL(do_QDIO); 1544 1545 /** 1546 * qdio_start_irq - enable interrupt processing for the device 1547 * @cdev: associated ccw_device for the qdio subchannel 1548 * 1549 * Return codes 1550 * 0 - success 1551 * 1 - irqs not started since new data is available 1552 */ 1553 int qdio_start_irq(struct ccw_device *cdev) 1554 { 1555 struct qdio_q *q; 1556 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1557 unsigned int i; 1558 1559 if (!irq_ptr) 1560 return -ENODEV; 1561 1562 for_each_input_queue(irq_ptr, q, i) 1563 qdio_stop_polling(q); 1564 1565 clear_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state); 1566 1567 /* 1568 * We need to check again to not lose initiative after 1569 * resetting the ACK state. 1570 */ 1571 if (test_nonshared_ind(irq_ptr)) 1572 goto rescan; 1573 1574 for_each_input_queue(irq_ptr, q, i) { 1575 if (!qdio_inbound_q_done(q, q->first_to_check)) 1576 goto rescan; 1577 } 1578 1579 return 0; 1580 1581 rescan: 1582 if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state)) 1583 return 0; 1584 else 1585 return 1; 1586 1587 } 1588 EXPORT_SYMBOL(qdio_start_irq); 1589 1590 static int __qdio_inspect_queue(struct qdio_q *q, unsigned int *bufnr, 1591 unsigned int *error) 1592 { 1593 unsigned int start = q->first_to_check; 1594 int count; 1595 1596 count = q->is_input_q ? qdio_inbound_q_moved(q, start) : 1597 qdio_outbound_q_moved(q, start); 1598 if (count == 0) 1599 return 0; 1600 1601 *bufnr = start; 1602 *error = q->qdio_error; 1603 1604 /* for the next time */ 1605 q->first_to_check = add_buf(start, count); 1606 q->qdio_error = 0; 1607 1608 return count; 1609 } 1610 1611 int qdio_inspect_queue(struct ccw_device *cdev, unsigned int nr, bool is_input, 1612 unsigned int *bufnr, unsigned int *error) 1613 { 1614 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1615 struct qdio_q *q; 1616 1617 if (!irq_ptr) 1618 return -ENODEV; 1619 q = is_input ? irq_ptr->input_qs[nr] : irq_ptr->output_qs[nr]; 1620 1621 if (need_siga_sync(q)) 1622 qdio_siga_sync_q(q); 1623 1624 return __qdio_inspect_queue(q, bufnr, error); 1625 } 1626 EXPORT_SYMBOL_GPL(qdio_inspect_queue); 1627 1628 /** 1629 * qdio_get_next_buffers - process input buffers 1630 * @cdev: associated ccw_device for the qdio subchannel 1631 * @nr: input queue number 1632 * @bufnr: first filled buffer number 1633 * @error: buffers are in error state 1634 * 1635 * Return codes 1636 * < 0 - error 1637 * = 0 - no new buffers found 1638 * > 0 - number of processed buffers 1639 */ 1640 int qdio_get_next_buffers(struct ccw_device *cdev, int nr, int *bufnr, 1641 int *error) 1642 { 1643 struct qdio_q *q; 1644 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1645 1646 if (!irq_ptr) 1647 return -ENODEV; 1648 q = irq_ptr->input_qs[nr]; 1649 1650 /* 1651 * Cannot rely on automatic sync after interrupt since queues may 1652 * also be examined without interrupt. 1653 */ 1654 if (need_siga_sync(q)) 1655 qdio_sync_queues(q); 1656 1657 qdio_check_outbound_pci_queues(irq_ptr); 1658 1659 /* Note: upper-layer MUST stop processing immediately here ... */ 1660 if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)) 1661 return -EIO; 1662 1663 return __qdio_inspect_queue(q, bufnr, error); 1664 } 1665 EXPORT_SYMBOL(qdio_get_next_buffers); 1666 1667 /** 1668 * qdio_stop_irq - disable interrupt processing for the device 1669 * @cdev: associated ccw_device for the qdio subchannel 1670 * 1671 * Return codes 1672 * 0 - interrupts were already disabled 1673 * 1 - interrupts successfully disabled 1674 */ 1675 int qdio_stop_irq(struct ccw_device *cdev) 1676 { 1677 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1678 1679 if (!irq_ptr) 1680 return -ENODEV; 1681 1682 if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state)) 1683 return 0; 1684 else 1685 return 1; 1686 } 1687 EXPORT_SYMBOL(qdio_stop_irq); 1688 1689 static int __init init_QDIO(void) 1690 { 1691 int rc; 1692 1693 rc = qdio_debug_init(); 1694 if (rc) 1695 return rc; 1696 rc = qdio_setup_init(); 1697 if (rc) 1698 goto out_debug; 1699 rc = qdio_thinint_init(); 1700 if (rc) 1701 goto out_cache; 1702 return 0; 1703 1704 out_cache: 1705 qdio_setup_exit(); 1706 out_debug: 1707 qdio_debug_exit(); 1708 return rc; 1709 } 1710 1711 static void __exit exit_QDIO(void) 1712 { 1713 qdio_thinint_exit(); 1714 qdio_setup_exit(); 1715 qdio_debug_exit(); 1716 } 1717 1718 module_init(init_QDIO); 1719 module_exit(exit_QDIO); 1720