1 /* 2 * linux/drivers/s390/cio/qdio_main.c 3 * 4 * Linux for s390 qdio support, buffer handling, qdio API and module support. 5 * 6 * Copyright 2000,2008 IBM Corp. 7 * Author(s): Utz Bacher <utz.bacher@de.ibm.com> 8 * Jan Glauber <jang@linux.vnet.ibm.com> 9 * 2.6 cio integration by Cornelia Huck <cornelia.huck@de.ibm.com> 10 */ 11 #include <linux/module.h> 12 #include <linux/init.h> 13 #include <linux/kernel.h> 14 #include <linux/timer.h> 15 #include <linux/delay.h> 16 #include <linux/gfp.h> 17 #include <asm/atomic.h> 18 #include <asm/debug.h> 19 #include <asm/qdio.h> 20 21 #include "cio.h" 22 #include "css.h" 23 #include "device.h" 24 #include "qdio.h" 25 #include "qdio_debug.h" 26 27 MODULE_AUTHOR("Utz Bacher <utz.bacher@de.ibm.com>,"\ 28 "Jan Glauber <jang@linux.vnet.ibm.com>"); 29 MODULE_DESCRIPTION("QDIO base support"); 30 MODULE_LICENSE("GPL"); 31 32 static inline int do_siga_sync(struct subchannel_id schid, 33 unsigned int out_mask, unsigned int in_mask) 34 { 35 register unsigned long __fc asm ("0") = 2; 36 register struct subchannel_id __schid asm ("1") = schid; 37 register unsigned long out asm ("2") = out_mask; 38 register unsigned long in asm ("3") = in_mask; 39 int cc; 40 41 asm volatile( 42 " siga 0\n" 43 " ipm %0\n" 44 " srl %0,28\n" 45 : "=d" (cc) 46 : "d" (__fc), "d" (__schid), "d" (out), "d" (in) : "cc"); 47 return cc; 48 } 49 50 static inline int do_siga_input(struct subchannel_id schid, unsigned int mask) 51 { 52 register unsigned long __fc asm ("0") = 1; 53 register struct subchannel_id __schid asm ("1") = schid; 54 register unsigned long __mask asm ("2") = mask; 55 int cc; 56 57 asm volatile( 58 " siga 0\n" 59 " ipm %0\n" 60 " srl %0,28\n" 61 : "=d" (cc) 62 : "d" (__fc), "d" (__schid), "d" (__mask) : "cc", "memory"); 63 return cc; 64 } 65 66 /** 67 * do_siga_output - perform SIGA-w/wt function 68 * @schid: subchannel id or in case of QEBSM the subchannel token 69 * @mask: which output queues to process 70 * @bb: busy bit indicator, set only if SIGA-w/wt could not access a buffer 71 * @fc: function code to perform 72 * 73 * Returns cc or QDIO_ERROR_SIGA_ACCESS_EXCEPTION. 74 * Note: For IQDC unicast queues only the highest priority queue is processed. 75 */ 76 static inline int do_siga_output(unsigned long schid, unsigned long mask, 77 unsigned int *bb, unsigned int fc) 78 { 79 register unsigned long __fc asm("0") = fc; 80 register unsigned long __schid asm("1") = schid; 81 register unsigned long __mask asm("2") = mask; 82 int cc = QDIO_ERROR_SIGA_ACCESS_EXCEPTION; 83 84 asm volatile( 85 " siga 0\n" 86 "0: ipm %0\n" 87 " srl %0,28\n" 88 "1:\n" 89 EX_TABLE(0b, 1b) 90 : "+d" (cc), "+d" (__fc), "+d" (__schid), "+d" (__mask) 91 : : "cc", "memory"); 92 *bb = ((unsigned int) __fc) >> 31; 93 return cc; 94 } 95 96 static inline int qdio_check_ccq(struct qdio_q *q, unsigned int ccq) 97 { 98 /* all done or next buffer state different */ 99 if (ccq == 0 || ccq == 32) 100 return 0; 101 /* not all buffers processed */ 102 if (ccq == 96 || ccq == 97) 103 return 1; 104 /* notify devices immediately */ 105 DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq); 106 return -EIO; 107 } 108 109 /** 110 * qdio_do_eqbs - extract buffer states for QEBSM 111 * @q: queue to manipulate 112 * @state: state of the extracted buffers 113 * @start: buffer number to start at 114 * @count: count of buffers to examine 115 * @auto_ack: automatically acknowledge buffers 116 * 117 * Returns the number of successfully extracted equal buffer states. 118 * Stops processing if a state is different from the last buffers state. 119 */ 120 static int qdio_do_eqbs(struct qdio_q *q, unsigned char *state, 121 int start, int count, int auto_ack) 122 { 123 unsigned int ccq = 0; 124 int tmp_count = count, tmp_start = start; 125 int nr = q->nr; 126 int rc; 127 128 BUG_ON(!q->irq_ptr->sch_token); 129 qperf_inc(q, eqbs); 130 131 if (!q->is_input_q) 132 nr += q->irq_ptr->nr_input_qs; 133 again: 134 ccq = do_eqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count, 135 auto_ack); 136 rc = qdio_check_ccq(q, ccq); 137 138 /* At least one buffer was processed, return and extract the remaining 139 * buffers later. 140 */ 141 if ((ccq == 96) && (count != tmp_count)) { 142 qperf_inc(q, eqbs_partial); 143 return (count - tmp_count); 144 } 145 146 if (rc == 1) { 147 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS again:%2d", ccq); 148 goto again; 149 } 150 151 if (rc < 0) { 152 DBF_ERROR("%4x EQBS ERROR", SCH_NO(q)); 153 DBF_ERROR("%3d%3d%2d", count, tmp_count, nr); 154 q->handler(q->irq_ptr->cdev, 155 QDIO_ERROR_ACTIVATE_CHECK_CONDITION, 156 0, -1, -1, q->irq_ptr->int_parm); 157 return 0; 158 } 159 return count - tmp_count; 160 } 161 162 /** 163 * qdio_do_sqbs - set buffer states for QEBSM 164 * @q: queue to manipulate 165 * @state: new state of the buffers 166 * @start: first buffer number to change 167 * @count: how many buffers to change 168 * 169 * Returns the number of successfully changed buffers. 170 * Does retrying until the specified count of buffer states is set or an 171 * error occurs. 172 */ 173 static int qdio_do_sqbs(struct qdio_q *q, unsigned char state, int start, 174 int count) 175 { 176 unsigned int ccq = 0; 177 int tmp_count = count, tmp_start = start; 178 int nr = q->nr; 179 int rc; 180 181 if (!count) 182 return 0; 183 184 BUG_ON(!q->irq_ptr->sch_token); 185 qperf_inc(q, sqbs); 186 187 if (!q->is_input_q) 188 nr += q->irq_ptr->nr_input_qs; 189 again: 190 ccq = do_sqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count); 191 rc = qdio_check_ccq(q, ccq); 192 if (rc == 1) { 193 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "SQBS again:%2d", ccq); 194 qperf_inc(q, sqbs_partial); 195 goto again; 196 } 197 if (rc < 0) { 198 DBF_ERROR("%4x SQBS ERROR", SCH_NO(q)); 199 DBF_ERROR("%3d%3d%2d", count, tmp_count, nr); 200 q->handler(q->irq_ptr->cdev, 201 QDIO_ERROR_ACTIVATE_CHECK_CONDITION, 202 0, -1, -1, q->irq_ptr->int_parm); 203 return 0; 204 } 205 WARN_ON(tmp_count); 206 return count - tmp_count; 207 } 208 209 /* returns number of examined buffers and their common state in *state */ 210 static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr, 211 unsigned char *state, unsigned int count, 212 int auto_ack) 213 { 214 unsigned char __state = 0; 215 int i; 216 217 BUG_ON(bufnr > QDIO_MAX_BUFFERS_MASK); 218 BUG_ON(count > QDIO_MAX_BUFFERS_PER_Q); 219 220 if (is_qebsm(q)) 221 return qdio_do_eqbs(q, state, bufnr, count, auto_ack); 222 223 for (i = 0; i < count; i++) { 224 if (!__state) 225 __state = q->slsb.val[bufnr]; 226 else if (q->slsb.val[bufnr] != __state) 227 break; 228 bufnr = next_buf(bufnr); 229 } 230 *state = __state; 231 return i; 232 } 233 234 static inline int get_buf_state(struct qdio_q *q, unsigned int bufnr, 235 unsigned char *state, int auto_ack) 236 { 237 return get_buf_states(q, bufnr, state, 1, auto_ack); 238 } 239 240 /* wrap-around safe setting of slsb states, returns number of changed buffers */ 241 static inline int set_buf_states(struct qdio_q *q, int bufnr, 242 unsigned char state, int count) 243 { 244 int i; 245 246 BUG_ON(bufnr > QDIO_MAX_BUFFERS_MASK); 247 BUG_ON(count > QDIO_MAX_BUFFERS_PER_Q); 248 249 if (is_qebsm(q)) 250 return qdio_do_sqbs(q, state, bufnr, count); 251 252 for (i = 0; i < count; i++) { 253 xchg(&q->slsb.val[bufnr], state); 254 bufnr = next_buf(bufnr); 255 } 256 return count; 257 } 258 259 static inline int set_buf_state(struct qdio_q *q, int bufnr, 260 unsigned char state) 261 { 262 return set_buf_states(q, bufnr, state, 1); 263 } 264 265 /* set slsb states to initial state */ 266 void qdio_init_buf_states(struct qdio_irq *irq_ptr) 267 { 268 struct qdio_q *q; 269 int i; 270 271 for_each_input_queue(irq_ptr, q, i) 272 set_buf_states(q, 0, SLSB_P_INPUT_NOT_INIT, 273 QDIO_MAX_BUFFERS_PER_Q); 274 for_each_output_queue(irq_ptr, q, i) 275 set_buf_states(q, 0, SLSB_P_OUTPUT_NOT_INIT, 276 QDIO_MAX_BUFFERS_PER_Q); 277 } 278 279 static inline int qdio_siga_sync(struct qdio_q *q, unsigned int output, 280 unsigned int input) 281 { 282 int cc; 283 284 if (!need_siga_sync(q)) 285 return 0; 286 287 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-s:%1d", q->nr); 288 qperf_inc(q, siga_sync); 289 290 cc = do_siga_sync(q->irq_ptr->schid, output, input); 291 if (cc) 292 DBF_ERROR("%4x SIGA-S:%2d", SCH_NO(q), cc); 293 return cc; 294 } 295 296 static inline int qdio_siga_sync_q(struct qdio_q *q) 297 { 298 if (q->is_input_q) 299 return qdio_siga_sync(q, 0, q->mask); 300 else 301 return qdio_siga_sync(q, q->mask, 0); 302 } 303 304 static inline int qdio_siga_sync_out(struct qdio_q *q) 305 { 306 return qdio_siga_sync(q, ~0U, 0); 307 } 308 309 static inline int qdio_siga_sync_all(struct qdio_q *q) 310 { 311 return qdio_siga_sync(q, ~0U, ~0U); 312 } 313 314 static int qdio_siga_output(struct qdio_q *q, unsigned int *busy_bit) 315 { 316 unsigned long schid; 317 unsigned int fc = 0; 318 u64 start_time = 0; 319 int cc; 320 321 if (q->u.out.use_enh_siga) 322 fc = 3; 323 324 if (is_qebsm(q)) { 325 schid = q->irq_ptr->sch_token; 326 fc |= 0x80; 327 } 328 else 329 schid = *((u32 *)&q->irq_ptr->schid); 330 331 again: 332 cc = do_siga_output(schid, q->mask, busy_bit, fc); 333 334 /* hipersocket busy condition */ 335 if (*busy_bit) { 336 WARN_ON(queue_type(q) != QDIO_IQDIO_QFMT || cc != 2); 337 338 if (!start_time) { 339 start_time = get_clock(); 340 goto again; 341 } 342 if ((get_clock() - start_time) < QDIO_BUSY_BIT_PATIENCE) 343 goto again; 344 } 345 return cc; 346 } 347 348 static inline int qdio_siga_input(struct qdio_q *q) 349 { 350 int cc; 351 352 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-r:%1d", q->nr); 353 qperf_inc(q, siga_read); 354 355 cc = do_siga_input(q->irq_ptr->schid, q->mask); 356 if (cc) 357 DBF_ERROR("%4x SIGA-R:%2d", SCH_NO(q), cc); 358 return cc; 359 } 360 361 static inline void qdio_sync_after_thinint(struct qdio_q *q) 362 { 363 if (pci_out_supported(q)) { 364 if (need_siga_sync_thinint(q)) 365 qdio_siga_sync_all(q); 366 else if (need_siga_sync_out_thinint(q)) 367 qdio_siga_sync_out(q); 368 } else 369 qdio_siga_sync_q(q); 370 } 371 372 int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr, 373 unsigned char *state) 374 { 375 qdio_siga_sync_q(q); 376 return get_buf_states(q, bufnr, state, 1, 0); 377 } 378 379 static inline void qdio_stop_polling(struct qdio_q *q) 380 { 381 if (!q->u.in.polling) 382 return; 383 384 q->u.in.polling = 0; 385 qperf_inc(q, stop_polling); 386 387 /* show the card that we are not polling anymore */ 388 if (is_qebsm(q)) { 389 set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT, 390 q->u.in.ack_count); 391 q->u.in.ack_count = 0; 392 } else 393 set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT); 394 } 395 396 static inline void account_sbals(struct qdio_q *q, int count) 397 { 398 int pos = 0; 399 400 q->q_stats.nr_sbal_total += count; 401 if (count == QDIO_MAX_BUFFERS_MASK) { 402 q->q_stats.nr_sbals[7]++; 403 return; 404 } 405 while (count >>= 1) 406 pos++; 407 q->q_stats.nr_sbals[pos]++; 408 } 409 410 static void announce_buffer_error(struct qdio_q *q, int count) 411 { 412 q->qdio_error |= QDIO_ERROR_SLSB_STATE; 413 414 /* special handling for no target buffer empty */ 415 if ((!q->is_input_q && 416 (q->sbal[q->first_to_check]->element[15].flags & 0xff) == 0x10)) { 417 qperf_inc(q, target_full); 418 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x", 419 q->first_to_check); 420 return; 421 } 422 423 DBF_ERROR("%4x BUF ERROR", SCH_NO(q)); 424 DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr); 425 DBF_ERROR("FTC:%3d C:%3d", q->first_to_check, count); 426 DBF_ERROR("F14:%2x F15:%2x", 427 q->sbal[q->first_to_check]->element[14].flags & 0xff, 428 q->sbal[q->first_to_check]->element[15].flags & 0xff); 429 } 430 431 static inline void inbound_primed(struct qdio_q *q, int count) 432 { 433 int new; 434 435 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim: %02x", count); 436 437 /* for QEBSM the ACK was already set by EQBS */ 438 if (is_qebsm(q)) { 439 if (!q->u.in.polling) { 440 q->u.in.polling = 1; 441 q->u.in.ack_count = count; 442 q->u.in.ack_start = q->first_to_check; 443 return; 444 } 445 446 /* delete the previous ACK's */ 447 set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT, 448 q->u.in.ack_count); 449 q->u.in.ack_count = count; 450 q->u.in.ack_start = q->first_to_check; 451 return; 452 } 453 454 /* 455 * ACK the newest buffer. The ACK will be removed in qdio_stop_polling 456 * or by the next inbound run. 457 */ 458 new = add_buf(q->first_to_check, count - 1); 459 if (q->u.in.polling) { 460 /* reset the previous ACK but first set the new one */ 461 set_buf_state(q, new, SLSB_P_INPUT_ACK); 462 set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT); 463 } else { 464 q->u.in.polling = 1; 465 set_buf_state(q, new, SLSB_P_INPUT_ACK); 466 } 467 468 q->u.in.ack_start = new; 469 count--; 470 if (!count) 471 return; 472 /* need to change ALL buffers to get more interrupts */ 473 set_buf_states(q, q->first_to_check, SLSB_P_INPUT_NOT_INIT, count); 474 } 475 476 static int get_inbound_buffer_frontier(struct qdio_q *q) 477 { 478 int count, stop; 479 unsigned char state; 480 481 /* 482 * Don't check 128 buffers, as otherwise qdio_inbound_q_moved 483 * would return 0. 484 */ 485 count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK); 486 stop = add_buf(q->first_to_check, count); 487 488 if (q->first_to_check == stop) 489 goto out; 490 491 /* 492 * No siga sync here, as a PCI or we after a thin interrupt 493 * already sync'ed the queues. 494 */ 495 count = get_buf_states(q, q->first_to_check, &state, count, 1); 496 if (!count) 497 goto out; 498 499 switch (state) { 500 case SLSB_P_INPUT_PRIMED: 501 inbound_primed(q, count); 502 q->first_to_check = add_buf(q->first_to_check, count); 503 if (atomic_sub(count, &q->nr_buf_used) == 0) 504 qperf_inc(q, inbound_queue_full); 505 if (q->irq_ptr->perf_stat_enabled) 506 account_sbals(q, count); 507 break; 508 case SLSB_P_INPUT_ERROR: 509 announce_buffer_error(q, count); 510 /* process the buffer, the upper layer will take care of it */ 511 q->first_to_check = add_buf(q->first_to_check, count); 512 atomic_sub(count, &q->nr_buf_used); 513 if (q->irq_ptr->perf_stat_enabled) 514 account_sbals_error(q, count); 515 break; 516 case SLSB_CU_INPUT_EMPTY: 517 case SLSB_P_INPUT_NOT_INIT: 518 case SLSB_P_INPUT_ACK: 519 if (q->irq_ptr->perf_stat_enabled) 520 q->q_stats.nr_sbal_nop++; 521 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop"); 522 break; 523 default: 524 BUG(); 525 } 526 out: 527 return q->first_to_check; 528 } 529 530 static int qdio_inbound_q_moved(struct qdio_q *q) 531 { 532 int bufnr; 533 534 bufnr = get_inbound_buffer_frontier(q); 535 536 if ((bufnr != q->last_move) || q->qdio_error) { 537 q->last_move = bufnr; 538 if (!is_thinint_irq(q->irq_ptr) && MACHINE_IS_LPAR) 539 q->u.in.timestamp = get_clock(); 540 return 1; 541 } else 542 return 0; 543 } 544 545 static inline int qdio_inbound_q_done(struct qdio_q *q) 546 { 547 unsigned char state = 0; 548 549 if (!atomic_read(&q->nr_buf_used)) 550 return 1; 551 552 qdio_siga_sync_q(q); 553 get_buf_state(q, q->first_to_check, &state, 0); 554 555 if (state == SLSB_P_INPUT_PRIMED || state == SLSB_P_INPUT_ERROR) 556 /* more work coming */ 557 return 0; 558 559 if (is_thinint_irq(q->irq_ptr)) 560 return 1; 561 562 /* don't poll under z/VM */ 563 if (MACHINE_IS_VM) 564 return 1; 565 566 /* 567 * At this point we know, that inbound first_to_check 568 * has (probably) not moved (see qdio_inbound_processing). 569 */ 570 if (get_clock() > q->u.in.timestamp + QDIO_INPUT_THRESHOLD) { 571 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in done:%02x", 572 q->first_to_check); 573 return 1; 574 } else 575 return 0; 576 } 577 578 static void qdio_kick_handler(struct qdio_q *q) 579 { 580 int start = q->first_to_kick; 581 int end = q->first_to_check; 582 int count; 583 584 if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)) 585 return; 586 587 count = sub_buf(end, start); 588 589 if (q->is_input_q) { 590 qperf_inc(q, inbound_handler); 591 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "kih s:%02x c:%02x", start, count); 592 } else { 593 qperf_inc(q, outbound_handler); 594 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "koh: s:%02x c:%02x", 595 start, count); 596 } 597 598 q->handler(q->irq_ptr->cdev, q->qdio_error, q->nr, start, count, 599 q->irq_ptr->int_parm); 600 601 /* for the next time */ 602 q->first_to_kick = end; 603 q->qdio_error = 0; 604 } 605 606 static void __qdio_inbound_processing(struct qdio_q *q) 607 { 608 qperf_inc(q, tasklet_inbound); 609 610 if (!qdio_inbound_q_moved(q)) 611 return; 612 613 qdio_kick_handler(q); 614 615 if (!qdio_inbound_q_done(q)) { 616 /* means poll time is not yet over */ 617 qperf_inc(q, tasklet_inbound_resched); 618 if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) { 619 tasklet_schedule(&q->tasklet); 620 return; 621 } 622 } 623 624 qdio_stop_polling(q); 625 /* 626 * We need to check again to not lose initiative after 627 * resetting the ACK state. 628 */ 629 if (!qdio_inbound_q_done(q)) { 630 qperf_inc(q, tasklet_inbound_resched2); 631 if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) 632 tasklet_schedule(&q->tasklet); 633 } 634 } 635 636 void qdio_inbound_processing(unsigned long data) 637 { 638 struct qdio_q *q = (struct qdio_q *)data; 639 __qdio_inbound_processing(q); 640 } 641 642 static int get_outbound_buffer_frontier(struct qdio_q *q) 643 { 644 int count, stop; 645 unsigned char state; 646 647 if (((queue_type(q) != QDIO_IQDIO_QFMT) && !pci_out_supported(q)) || 648 (queue_type(q) == QDIO_IQDIO_QFMT && multicast_outbound(q))) 649 qdio_siga_sync_q(q); 650 651 /* 652 * Don't check 128 buffers, as otherwise qdio_inbound_q_moved 653 * would return 0. 654 */ 655 count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK); 656 stop = add_buf(q->first_to_check, count); 657 658 if (q->first_to_check == stop) 659 return q->first_to_check; 660 661 count = get_buf_states(q, q->first_to_check, &state, count, 0); 662 if (!count) 663 return q->first_to_check; 664 665 switch (state) { 666 case SLSB_P_OUTPUT_EMPTY: 667 /* the adapter got it */ 668 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out empty:%1d %02x", q->nr, count); 669 670 atomic_sub(count, &q->nr_buf_used); 671 q->first_to_check = add_buf(q->first_to_check, count); 672 if (q->irq_ptr->perf_stat_enabled) 673 account_sbals(q, count); 674 break; 675 case SLSB_P_OUTPUT_ERROR: 676 announce_buffer_error(q, count); 677 /* process the buffer, the upper layer will take care of it */ 678 q->first_to_check = add_buf(q->first_to_check, count); 679 atomic_sub(count, &q->nr_buf_used); 680 if (q->irq_ptr->perf_stat_enabled) 681 account_sbals_error(q, count); 682 break; 683 case SLSB_CU_OUTPUT_PRIMED: 684 /* the adapter has not fetched the output yet */ 685 if (q->irq_ptr->perf_stat_enabled) 686 q->q_stats.nr_sbal_nop++; 687 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d", q->nr); 688 break; 689 case SLSB_P_OUTPUT_NOT_INIT: 690 case SLSB_P_OUTPUT_HALTED: 691 break; 692 default: 693 BUG(); 694 } 695 return q->first_to_check; 696 } 697 698 /* all buffers processed? */ 699 static inline int qdio_outbound_q_done(struct qdio_q *q) 700 { 701 return atomic_read(&q->nr_buf_used) == 0; 702 } 703 704 static inline int qdio_outbound_q_moved(struct qdio_q *q) 705 { 706 int bufnr; 707 708 bufnr = get_outbound_buffer_frontier(q); 709 710 if ((bufnr != q->last_move) || q->qdio_error) { 711 q->last_move = bufnr; 712 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out moved:%1d", q->nr); 713 return 1; 714 } else 715 return 0; 716 } 717 718 static int qdio_kick_outbound_q(struct qdio_q *q) 719 { 720 unsigned int busy_bit; 721 int cc; 722 723 if (!need_siga_out(q)) 724 return 0; 725 726 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w:%1d", q->nr); 727 qperf_inc(q, siga_write); 728 729 cc = qdio_siga_output(q, &busy_bit); 730 switch (cc) { 731 case 0: 732 break; 733 case 2: 734 if (busy_bit) { 735 DBF_ERROR("%4x cc2 REP:%1d", SCH_NO(q), q->nr); 736 cc |= QDIO_ERROR_SIGA_BUSY; 737 } else 738 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w cc2:%1d", q->nr); 739 break; 740 case 1: 741 case 3: 742 DBF_ERROR("%4x SIGA-W:%1d", SCH_NO(q), cc); 743 break; 744 } 745 return cc; 746 } 747 748 static void __qdio_outbound_processing(struct qdio_q *q) 749 { 750 qperf_inc(q, tasklet_outbound); 751 BUG_ON(atomic_read(&q->nr_buf_used) < 0); 752 753 if (qdio_outbound_q_moved(q)) 754 qdio_kick_handler(q); 755 756 if (queue_type(q) == QDIO_ZFCP_QFMT) 757 if (!pci_out_supported(q) && !qdio_outbound_q_done(q)) 758 goto sched; 759 760 /* bail out for HiperSockets unicast queues */ 761 if (queue_type(q) == QDIO_IQDIO_QFMT && !multicast_outbound(q)) 762 return; 763 764 if ((queue_type(q) == QDIO_IQDIO_QFMT) && 765 (atomic_read(&q->nr_buf_used)) > QDIO_IQDIO_POLL_LVL) 766 goto sched; 767 768 if (q->u.out.pci_out_enabled) 769 return; 770 771 /* 772 * Now we know that queue type is either qeth without pci enabled 773 * or HiperSockets multicast. Make sure buffer switch from PRIMED to 774 * EMPTY is noticed and outbound_handler is called after some time. 775 */ 776 if (qdio_outbound_q_done(q)) 777 del_timer(&q->u.out.timer); 778 else 779 if (!timer_pending(&q->u.out.timer)) 780 mod_timer(&q->u.out.timer, jiffies + 10 * HZ); 781 return; 782 783 sched: 784 if (unlikely(q->irq_ptr->state == QDIO_IRQ_STATE_STOPPED)) 785 return; 786 tasklet_schedule(&q->tasklet); 787 } 788 789 /* outbound tasklet */ 790 void qdio_outbound_processing(unsigned long data) 791 { 792 struct qdio_q *q = (struct qdio_q *)data; 793 __qdio_outbound_processing(q); 794 } 795 796 void qdio_outbound_timer(unsigned long data) 797 { 798 struct qdio_q *q = (struct qdio_q *)data; 799 800 if (unlikely(q->irq_ptr->state == QDIO_IRQ_STATE_STOPPED)) 801 return; 802 tasklet_schedule(&q->tasklet); 803 } 804 805 static inline void qdio_check_outbound_after_thinint(struct qdio_q *q) 806 { 807 struct qdio_q *out; 808 int i; 809 810 if (!pci_out_supported(q)) 811 return; 812 813 for_each_output_queue(q->irq_ptr, out, i) 814 if (!qdio_outbound_q_done(out)) 815 tasklet_schedule(&out->tasklet); 816 } 817 818 static void __tiqdio_inbound_processing(struct qdio_q *q) 819 { 820 qperf_inc(q, tasklet_inbound); 821 qdio_sync_after_thinint(q); 822 823 /* 824 * The interrupt could be caused by a PCI request. Check the 825 * PCI capable outbound queues. 826 */ 827 qdio_check_outbound_after_thinint(q); 828 829 if (!qdio_inbound_q_moved(q)) 830 return; 831 832 qdio_kick_handler(q); 833 834 if (!qdio_inbound_q_done(q)) { 835 qperf_inc(q, tasklet_inbound_resched); 836 if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) { 837 tasklet_schedule(&q->tasklet); 838 return; 839 } 840 } 841 842 qdio_stop_polling(q); 843 /* 844 * We need to check again to not lose initiative after 845 * resetting the ACK state. 846 */ 847 if (!qdio_inbound_q_done(q)) { 848 qperf_inc(q, tasklet_inbound_resched2); 849 if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) 850 tasklet_schedule(&q->tasklet); 851 } 852 } 853 854 void tiqdio_inbound_processing(unsigned long data) 855 { 856 struct qdio_q *q = (struct qdio_q *)data; 857 __tiqdio_inbound_processing(q); 858 } 859 860 static inline void qdio_set_state(struct qdio_irq *irq_ptr, 861 enum qdio_irq_states state) 862 { 863 DBF_DEV_EVENT(DBF_INFO, irq_ptr, "newstate: %1d", state); 864 865 irq_ptr->state = state; 866 mb(); 867 } 868 869 static void qdio_irq_check_sense(struct qdio_irq *irq_ptr, struct irb *irb) 870 { 871 if (irb->esw.esw0.erw.cons) { 872 DBF_ERROR("%4x sense:", irq_ptr->schid.sch_no); 873 DBF_ERROR_HEX(irb, 64); 874 DBF_ERROR_HEX(irb->ecw, 64); 875 } 876 } 877 878 /* PCI interrupt handler */ 879 static void qdio_int_handler_pci(struct qdio_irq *irq_ptr) 880 { 881 int i; 882 struct qdio_q *q; 883 884 if (unlikely(irq_ptr->state == QDIO_IRQ_STATE_STOPPED)) 885 return; 886 887 for_each_input_queue(irq_ptr, q, i) { 888 if (q->u.in.queue_start_poll) { 889 /* skip if polling is enabled or already in work */ 890 if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED, 891 &q->u.in.queue_irq_state)) { 892 qperf_inc(q, int_discarded); 893 continue; 894 } 895 q->u.in.queue_start_poll(q->irq_ptr->cdev, q->nr, 896 q->irq_ptr->int_parm); 897 } else 898 tasklet_schedule(&q->tasklet); 899 } 900 901 if (!(irq_ptr->qib.ac & QIB_AC_OUTBOUND_PCI_SUPPORTED)) 902 return; 903 904 for_each_output_queue(irq_ptr, q, i) { 905 if (qdio_outbound_q_done(q)) 906 continue; 907 908 if (!siga_syncs_out_pci(q)) 909 qdio_siga_sync_q(q); 910 911 tasklet_schedule(&q->tasklet); 912 } 913 } 914 915 static void qdio_handle_activate_check(struct ccw_device *cdev, 916 unsigned long intparm, int cstat, int dstat) 917 { 918 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 919 struct qdio_q *q; 920 921 DBF_ERROR("%4x ACT CHECK", irq_ptr->schid.sch_no); 922 DBF_ERROR("intp :%lx", intparm); 923 DBF_ERROR("ds: %2x cs:%2x", dstat, cstat); 924 925 if (irq_ptr->nr_input_qs) { 926 q = irq_ptr->input_qs[0]; 927 } else if (irq_ptr->nr_output_qs) { 928 q = irq_ptr->output_qs[0]; 929 } else { 930 dump_stack(); 931 goto no_handler; 932 } 933 q->handler(q->irq_ptr->cdev, QDIO_ERROR_ACTIVATE_CHECK_CONDITION, 934 0, -1, -1, irq_ptr->int_parm); 935 no_handler: 936 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED); 937 } 938 939 static void qdio_establish_handle_irq(struct ccw_device *cdev, int cstat, 940 int dstat) 941 { 942 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 943 944 DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qest irq"); 945 946 if (cstat) 947 goto error; 948 if (dstat & ~(DEV_STAT_DEV_END | DEV_STAT_CHN_END)) 949 goto error; 950 if (!(dstat & DEV_STAT_DEV_END)) 951 goto error; 952 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ESTABLISHED); 953 return; 954 955 error: 956 DBF_ERROR("%4x EQ:error", irq_ptr->schid.sch_no); 957 DBF_ERROR("ds: %2x cs:%2x", dstat, cstat); 958 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR); 959 } 960 961 /* qdio interrupt handler */ 962 void qdio_int_handler(struct ccw_device *cdev, unsigned long intparm, 963 struct irb *irb) 964 { 965 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 966 int cstat, dstat; 967 968 if (!intparm || !irq_ptr) { 969 DBF_ERROR("qint:%4x", cdev->private->schid.sch_no); 970 return; 971 } 972 973 if (irq_ptr->perf_stat_enabled) 974 irq_ptr->perf_stat.qdio_int++; 975 976 if (IS_ERR(irb)) { 977 switch (PTR_ERR(irb)) { 978 case -EIO: 979 DBF_ERROR("%4x IO error", irq_ptr->schid.sch_no); 980 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR); 981 wake_up(&cdev->private->wait_q); 982 return; 983 default: 984 WARN_ON(1); 985 return; 986 } 987 } 988 qdio_irq_check_sense(irq_ptr, irb); 989 cstat = irb->scsw.cmd.cstat; 990 dstat = irb->scsw.cmd.dstat; 991 992 switch (irq_ptr->state) { 993 case QDIO_IRQ_STATE_INACTIVE: 994 qdio_establish_handle_irq(cdev, cstat, dstat); 995 break; 996 case QDIO_IRQ_STATE_CLEANUP: 997 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); 998 break; 999 case QDIO_IRQ_STATE_ESTABLISHED: 1000 case QDIO_IRQ_STATE_ACTIVE: 1001 if (cstat & SCHN_STAT_PCI) { 1002 qdio_int_handler_pci(irq_ptr); 1003 return; 1004 } 1005 if (cstat || dstat) 1006 qdio_handle_activate_check(cdev, intparm, cstat, 1007 dstat); 1008 break; 1009 case QDIO_IRQ_STATE_STOPPED: 1010 break; 1011 default: 1012 WARN_ON(1); 1013 } 1014 wake_up(&cdev->private->wait_q); 1015 } 1016 1017 /** 1018 * qdio_get_ssqd_desc - get qdio subchannel description 1019 * @cdev: ccw device to get description for 1020 * @data: where to store the ssqd 1021 * 1022 * Returns 0 or an error code. The results of the chsc are stored in the 1023 * specified structure. 1024 */ 1025 int qdio_get_ssqd_desc(struct ccw_device *cdev, 1026 struct qdio_ssqd_desc *data) 1027 { 1028 1029 if (!cdev || !cdev->private) 1030 return -EINVAL; 1031 1032 DBF_EVENT("get ssqd:%4x", cdev->private->schid.sch_no); 1033 return qdio_setup_get_ssqd(NULL, &cdev->private->schid, data); 1034 } 1035 EXPORT_SYMBOL_GPL(qdio_get_ssqd_desc); 1036 1037 static void qdio_shutdown_queues(struct ccw_device *cdev) 1038 { 1039 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1040 struct qdio_q *q; 1041 int i; 1042 1043 for_each_input_queue(irq_ptr, q, i) 1044 tasklet_kill(&q->tasklet); 1045 1046 for_each_output_queue(irq_ptr, q, i) { 1047 del_timer(&q->u.out.timer); 1048 tasklet_kill(&q->tasklet); 1049 } 1050 } 1051 1052 /** 1053 * qdio_shutdown - shut down a qdio subchannel 1054 * @cdev: associated ccw device 1055 * @how: use halt or clear to shutdown 1056 */ 1057 int qdio_shutdown(struct ccw_device *cdev, int how) 1058 { 1059 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1060 int rc; 1061 unsigned long flags; 1062 1063 if (!irq_ptr) 1064 return -ENODEV; 1065 1066 BUG_ON(irqs_disabled()); 1067 DBF_EVENT("qshutdown:%4x", cdev->private->schid.sch_no); 1068 1069 mutex_lock(&irq_ptr->setup_mutex); 1070 /* 1071 * Subchannel was already shot down. We cannot prevent being called 1072 * twice since cio may trigger a shutdown asynchronously. 1073 */ 1074 if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) { 1075 mutex_unlock(&irq_ptr->setup_mutex); 1076 return 0; 1077 } 1078 1079 /* 1080 * Indicate that the device is going down. Scheduling the queue 1081 * tasklets is forbidden from here on. 1082 */ 1083 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED); 1084 1085 tiqdio_remove_input_queues(irq_ptr); 1086 qdio_shutdown_queues(cdev); 1087 qdio_shutdown_debug_entries(irq_ptr, cdev); 1088 1089 /* cleanup subchannel */ 1090 spin_lock_irqsave(get_ccwdev_lock(cdev), flags); 1091 1092 if (how & QDIO_FLAG_CLEANUP_USING_CLEAR) 1093 rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP); 1094 else 1095 /* default behaviour is halt */ 1096 rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP); 1097 if (rc) { 1098 DBF_ERROR("%4x SHUTD ERR", irq_ptr->schid.sch_no); 1099 DBF_ERROR("rc:%4d", rc); 1100 goto no_cleanup; 1101 } 1102 1103 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_CLEANUP); 1104 spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags); 1105 wait_event_interruptible_timeout(cdev->private->wait_q, 1106 irq_ptr->state == QDIO_IRQ_STATE_INACTIVE || 1107 irq_ptr->state == QDIO_IRQ_STATE_ERR, 1108 10 * HZ); 1109 spin_lock_irqsave(get_ccwdev_lock(cdev), flags); 1110 1111 no_cleanup: 1112 qdio_shutdown_thinint(irq_ptr); 1113 1114 /* restore interrupt handler */ 1115 if ((void *)cdev->handler == (void *)qdio_int_handler) 1116 cdev->handler = irq_ptr->orig_handler; 1117 spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags); 1118 1119 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); 1120 mutex_unlock(&irq_ptr->setup_mutex); 1121 if (rc) 1122 return rc; 1123 return 0; 1124 } 1125 EXPORT_SYMBOL_GPL(qdio_shutdown); 1126 1127 /** 1128 * qdio_free - free data structures for a qdio subchannel 1129 * @cdev: associated ccw device 1130 */ 1131 int qdio_free(struct ccw_device *cdev) 1132 { 1133 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1134 1135 if (!irq_ptr) 1136 return -ENODEV; 1137 1138 DBF_EVENT("qfree:%4x", cdev->private->schid.sch_no); 1139 mutex_lock(&irq_ptr->setup_mutex); 1140 1141 if (irq_ptr->debug_area != NULL) { 1142 debug_unregister(irq_ptr->debug_area); 1143 irq_ptr->debug_area = NULL; 1144 } 1145 cdev->private->qdio_data = NULL; 1146 mutex_unlock(&irq_ptr->setup_mutex); 1147 1148 qdio_release_memory(irq_ptr); 1149 return 0; 1150 } 1151 EXPORT_SYMBOL_GPL(qdio_free); 1152 1153 /** 1154 * qdio_allocate - allocate qdio queues and associated data 1155 * @init_data: initialization data 1156 */ 1157 int qdio_allocate(struct qdio_initialize *init_data) 1158 { 1159 struct qdio_irq *irq_ptr; 1160 1161 DBF_EVENT("qallocate:%4x", init_data->cdev->private->schid.sch_no); 1162 1163 if ((init_data->no_input_qs && !init_data->input_handler) || 1164 (init_data->no_output_qs && !init_data->output_handler)) 1165 return -EINVAL; 1166 1167 if ((init_data->no_input_qs > QDIO_MAX_QUEUES_PER_IRQ) || 1168 (init_data->no_output_qs > QDIO_MAX_QUEUES_PER_IRQ)) 1169 return -EINVAL; 1170 1171 if ((!init_data->input_sbal_addr_array) || 1172 (!init_data->output_sbal_addr_array)) 1173 return -EINVAL; 1174 1175 /* irq_ptr must be in GFP_DMA since it contains ccw1.cda */ 1176 irq_ptr = (void *) get_zeroed_page(GFP_KERNEL | GFP_DMA); 1177 if (!irq_ptr) 1178 goto out_err; 1179 1180 mutex_init(&irq_ptr->setup_mutex); 1181 qdio_allocate_dbf(init_data, irq_ptr); 1182 1183 /* 1184 * Allocate a page for the chsc calls in qdio_establish. 1185 * Must be pre-allocated since a zfcp recovery will call 1186 * qdio_establish. In case of low memory and swap on a zfcp disk 1187 * we may not be able to allocate memory otherwise. 1188 */ 1189 irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL); 1190 if (!irq_ptr->chsc_page) 1191 goto out_rel; 1192 1193 /* qdr is used in ccw1.cda which is u32 */ 1194 irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL | GFP_DMA); 1195 if (!irq_ptr->qdr) 1196 goto out_rel; 1197 WARN_ON((unsigned long)irq_ptr->qdr & 0xfff); 1198 1199 if (qdio_allocate_qs(irq_ptr, init_data->no_input_qs, 1200 init_data->no_output_qs)) 1201 goto out_rel; 1202 1203 init_data->cdev->private->qdio_data = irq_ptr; 1204 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); 1205 return 0; 1206 out_rel: 1207 qdio_release_memory(irq_ptr); 1208 out_err: 1209 return -ENOMEM; 1210 } 1211 EXPORT_SYMBOL_GPL(qdio_allocate); 1212 1213 /** 1214 * qdio_establish - establish queues on a qdio subchannel 1215 * @init_data: initialization data 1216 */ 1217 int qdio_establish(struct qdio_initialize *init_data) 1218 { 1219 struct qdio_irq *irq_ptr; 1220 struct ccw_device *cdev = init_data->cdev; 1221 unsigned long saveflags; 1222 int rc; 1223 1224 DBF_EVENT("qestablish:%4x", cdev->private->schid.sch_no); 1225 1226 irq_ptr = cdev->private->qdio_data; 1227 if (!irq_ptr) 1228 return -ENODEV; 1229 1230 if (cdev->private->state != DEV_STATE_ONLINE) 1231 return -EINVAL; 1232 1233 mutex_lock(&irq_ptr->setup_mutex); 1234 qdio_setup_irq(init_data); 1235 1236 rc = qdio_establish_thinint(irq_ptr); 1237 if (rc) { 1238 mutex_unlock(&irq_ptr->setup_mutex); 1239 qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR); 1240 return rc; 1241 } 1242 1243 /* establish q */ 1244 irq_ptr->ccw.cmd_code = irq_ptr->equeue.cmd; 1245 irq_ptr->ccw.flags = CCW_FLAG_SLI; 1246 irq_ptr->ccw.count = irq_ptr->equeue.count; 1247 irq_ptr->ccw.cda = (u32)((addr_t)irq_ptr->qdr); 1248 1249 spin_lock_irqsave(get_ccwdev_lock(cdev), saveflags); 1250 ccw_device_set_options_mask(cdev, 0); 1251 1252 rc = ccw_device_start(cdev, &irq_ptr->ccw, QDIO_DOING_ESTABLISH, 0, 0); 1253 if (rc) { 1254 DBF_ERROR("%4x est IO ERR", irq_ptr->schid.sch_no); 1255 DBF_ERROR("rc:%4x", rc); 1256 } 1257 spin_unlock_irqrestore(get_ccwdev_lock(cdev), saveflags); 1258 1259 if (rc) { 1260 mutex_unlock(&irq_ptr->setup_mutex); 1261 qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR); 1262 return rc; 1263 } 1264 1265 wait_event_interruptible_timeout(cdev->private->wait_q, 1266 irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED || 1267 irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ); 1268 1269 if (irq_ptr->state != QDIO_IRQ_STATE_ESTABLISHED) { 1270 mutex_unlock(&irq_ptr->setup_mutex); 1271 qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR); 1272 return -EIO; 1273 } 1274 1275 qdio_setup_ssqd_info(irq_ptr); 1276 DBF_EVENT("qDmmwc:%2x", irq_ptr->ssqd_desc.mmwc); 1277 DBF_EVENT("qib ac:%4x", irq_ptr->qib.ac); 1278 1279 /* qebsm is now setup if available, initialize buffer states */ 1280 qdio_init_buf_states(irq_ptr); 1281 1282 mutex_unlock(&irq_ptr->setup_mutex); 1283 qdio_print_subchannel_info(irq_ptr, cdev); 1284 qdio_setup_debug_entries(irq_ptr, cdev); 1285 return 0; 1286 } 1287 EXPORT_SYMBOL_GPL(qdio_establish); 1288 1289 /** 1290 * qdio_activate - activate queues on a qdio subchannel 1291 * @cdev: associated cdev 1292 */ 1293 int qdio_activate(struct ccw_device *cdev) 1294 { 1295 struct qdio_irq *irq_ptr; 1296 int rc; 1297 unsigned long saveflags; 1298 1299 DBF_EVENT("qactivate:%4x", cdev->private->schid.sch_no); 1300 1301 irq_ptr = cdev->private->qdio_data; 1302 if (!irq_ptr) 1303 return -ENODEV; 1304 1305 if (cdev->private->state != DEV_STATE_ONLINE) 1306 return -EINVAL; 1307 1308 mutex_lock(&irq_ptr->setup_mutex); 1309 if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) { 1310 rc = -EBUSY; 1311 goto out; 1312 } 1313 1314 irq_ptr->ccw.cmd_code = irq_ptr->aqueue.cmd; 1315 irq_ptr->ccw.flags = CCW_FLAG_SLI; 1316 irq_ptr->ccw.count = irq_ptr->aqueue.count; 1317 irq_ptr->ccw.cda = 0; 1318 1319 spin_lock_irqsave(get_ccwdev_lock(cdev), saveflags); 1320 ccw_device_set_options(cdev, CCWDEV_REPORT_ALL); 1321 1322 rc = ccw_device_start(cdev, &irq_ptr->ccw, QDIO_DOING_ACTIVATE, 1323 0, DOIO_DENY_PREFETCH); 1324 if (rc) { 1325 DBF_ERROR("%4x act IO ERR", irq_ptr->schid.sch_no); 1326 DBF_ERROR("rc:%4x", rc); 1327 } 1328 spin_unlock_irqrestore(get_ccwdev_lock(cdev), saveflags); 1329 1330 if (rc) 1331 goto out; 1332 1333 if (is_thinint_irq(irq_ptr)) 1334 tiqdio_add_input_queues(irq_ptr); 1335 1336 /* wait for subchannel to become active */ 1337 msleep(5); 1338 1339 switch (irq_ptr->state) { 1340 case QDIO_IRQ_STATE_STOPPED: 1341 case QDIO_IRQ_STATE_ERR: 1342 rc = -EIO; 1343 break; 1344 default: 1345 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ACTIVE); 1346 rc = 0; 1347 } 1348 out: 1349 mutex_unlock(&irq_ptr->setup_mutex); 1350 return rc; 1351 } 1352 EXPORT_SYMBOL_GPL(qdio_activate); 1353 1354 static inline int buf_in_between(int bufnr, int start, int count) 1355 { 1356 int end = add_buf(start, count); 1357 1358 if (end > start) { 1359 if (bufnr >= start && bufnr < end) 1360 return 1; 1361 else 1362 return 0; 1363 } 1364 1365 /* wrap-around case */ 1366 if ((bufnr >= start && bufnr <= QDIO_MAX_BUFFERS_PER_Q) || 1367 (bufnr < end)) 1368 return 1; 1369 else 1370 return 0; 1371 } 1372 1373 /** 1374 * handle_inbound - reset processed input buffers 1375 * @q: queue containing the buffers 1376 * @callflags: flags 1377 * @bufnr: first buffer to process 1378 * @count: how many buffers are emptied 1379 */ 1380 static int handle_inbound(struct qdio_q *q, unsigned int callflags, 1381 int bufnr, int count) 1382 { 1383 int used, diff; 1384 1385 qperf_inc(q, inbound_call); 1386 1387 if (!q->u.in.polling) 1388 goto set; 1389 1390 /* protect against stop polling setting an ACK for an emptied slsb */ 1391 if (count == QDIO_MAX_BUFFERS_PER_Q) { 1392 /* overwriting everything, just delete polling status */ 1393 q->u.in.polling = 0; 1394 q->u.in.ack_count = 0; 1395 goto set; 1396 } else if (buf_in_between(q->u.in.ack_start, bufnr, count)) { 1397 if (is_qebsm(q)) { 1398 /* partial overwrite, just update ack_start */ 1399 diff = add_buf(bufnr, count); 1400 diff = sub_buf(diff, q->u.in.ack_start); 1401 q->u.in.ack_count -= diff; 1402 if (q->u.in.ack_count <= 0) { 1403 q->u.in.polling = 0; 1404 q->u.in.ack_count = 0; 1405 goto set; 1406 } 1407 q->u.in.ack_start = add_buf(q->u.in.ack_start, diff); 1408 } 1409 else 1410 /* the only ACK will be deleted, so stop polling */ 1411 q->u.in.polling = 0; 1412 } 1413 1414 set: 1415 count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count); 1416 1417 used = atomic_add_return(count, &q->nr_buf_used) - count; 1418 BUG_ON(used + count > QDIO_MAX_BUFFERS_PER_Q); 1419 1420 /* no need to signal as long as the adapter had free buffers */ 1421 if (used) 1422 return 0; 1423 1424 if (need_siga_in(q)) 1425 return qdio_siga_input(q); 1426 return 0; 1427 } 1428 1429 /** 1430 * handle_outbound - process filled outbound buffers 1431 * @q: queue containing the buffers 1432 * @callflags: flags 1433 * @bufnr: first buffer to process 1434 * @count: how many buffers are filled 1435 */ 1436 static int handle_outbound(struct qdio_q *q, unsigned int callflags, 1437 int bufnr, int count) 1438 { 1439 unsigned char state; 1440 int used, rc = 0; 1441 1442 qperf_inc(q, outbound_call); 1443 1444 count = set_buf_states(q, bufnr, SLSB_CU_OUTPUT_PRIMED, count); 1445 used = atomic_add_return(count, &q->nr_buf_used); 1446 BUG_ON(used > QDIO_MAX_BUFFERS_PER_Q); 1447 1448 if (callflags & QDIO_FLAG_PCI_OUT) { 1449 q->u.out.pci_out_enabled = 1; 1450 qperf_inc(q, pci_request_int); 1451 } 1452 else 1453 q->u.out.pci_out_enabled = 0; 1454 1455 if (queue_type(q) == QDIO_IQDIO_QFMT) { 1456 if (multicast_outbound(q)) 1457 rc = qdio_kick_outbound_q(q); 1458 else 1459 if ((q->irq_ptr->ssqd_desc.mmwc > 1) && 1460 (count > 1) && 1461 (count <= q->irq_ptr->ssqd_desc.mmwc)) { 1462 /* exploit enhanced SIGA */ 1463 q->u.out.use_enh_siga = 1; 1464 rc = qdio_kick_outbound_q(q); 1465 } else { 1466 /* 1467 * One siga-w per buffer required for unicast 1468 * HiperSockets. 1469 */ 1470 q->u.out.use_enh_siga = 0; 1471 while (count--) { 1472 rc = qdio_kick_outbound_q(q); 1473 if (rc) 1474 goto out; 1475 } 1476 } 1477 goto out; 1478 } 1479 1480 if (need_siga_sync(q)) { 1481 qdio_siga_sync_q(q); 1482 goto out; 1483 } 1484 1485 /* try to fast requeue buffers */ 1486 get_buf_state(q, prev_buf(bufnr), &state, 0); 1487 if (state != SLSB_CU_OUTPUT_PRIMED) 1488 rc = qdio_kick_outbound_q(q); 1489 else 1490 qperf_inc(q, fast_requeue); 1491 1492 out: 1493 tasklet_schedule(&q->tasklet); 1494 return rc; 1495 } 1496 1497 /** 1498 * do_QDIO - process input or output buffers 1499 * @cdev: associated ccw_device for the qdio subchannel 1500 * @callflags: input or output and special flags from the program 1501 * @q_nr: queue number 1502 * @bufnr: buffer number 1503 * @count: how many buffers to process 1504 */ 1505 int do_QDIO(struct ccw_device *cdev, unsigned int callflags, 1506 int q_nr, unsigned int bufnr, unsigned int count) 1507 { 1508 struct qdio_irq *irq_ptr; 1509 1510 if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q) 1511 return -EINVAL; 1512 1513 irq_ptr = cdev->private->qdio_data; 1514 if (!irq_ptr) 1515 return -ENODEV; 1516 1517 DBF_DEV_EVENT(DBF_INFO, irq_ptr, 1518 "do%02x b:%02x c:%02x", callflags, bufnr, count); 1519 1520 if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE) 1521 return -EBUSY; 1522 1523 if (callflags & QDIO_FLAG_SYNC_INPUT) 1524 return handle_inbound(irq_ptr->input_qs[q_nr], 1525 callflags, bufnr, count); 1526 else if (callflags & QDIO_FLAG_SYNC_OUTPUT) 1527 return handle_outbound(irq_ptr->output_qs[q_nr], 1528 callflags, bufnr, count); 1529 return -EINVAL; 1530 } 1531 EXPORT_SYMBOL_GPL(do_QDIO); 1532 1533 /** 1534 * qdio_start_irq - process input buffers 1535 * @cdev: associated ccw_device for the qdio subchannel 1536 * @nr: input queue number 1537 * 1538 * Return codes 1539 * 0 - success 1540 * 1 - irqs not started since new data is available 1541 */ 1542 int qdio_start_irq(struct ccw_device *cdev, int nr) 1543 { 1544 struct qdio_q *q; 1545 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1546 1547 if (!irq_ptr) 1548 return -ENODEV; 1549 q = irq_ptr->input_qs[nr]; 1550 1551 WARN_ON(queue_irqs_enabled(q)); 1552 1553 if (!shared_ind(q->irq_ptr)) 1554 xchg(q->irq_ptr->dsci, 0); 1555 1556 qdio_stop_polling(q); 1557 clear_bit(QDIO_QUEUE_IRQS_DISABLED, &q->u.in.queue_irq_state); 1558 1559 /* 1560 * We need to check again to not lose initiative after 1561 * resetting the ACK state. 1562 */ 1563 if (!shared_ind(q->irq_ptr) && *q->irq_ptr->dsci) 1564 goto rescan; 1565 if (!qdio_inbound_q_done(q)) 1566 goto rescan; 1567 return 0; 1568 1569 rescan: 1570 if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED, 1571 &q->u.in.queue_irq_state)) 1572 return 0; 1573 else 1574 return 1; 1575 1576 } 1577 EXPORT_SYMBOL(qdio_start_irq); 1578 1579 /** 1580 * qdio_get_next_buffers - process input buffers 1581 * @cdev: associated ccw_device for the qdio subchannel 1582 * @nr: input queue number 1583 * @bufnr: first filled buffer number 1584 * @error: buffers are in error state 1585 * 1586 * Return codes 1587 * < 0 - error 1588 * = 0 - no new buffers found 1589 * > 0 - number of processed buffers 1590 */ 1591 int qdio_get_next_buffers(struct ccw_device *cdev, int nr, int *bufnr, 1592 int *error) 1593 { 1594 struct qdio_q *q; 1595 int start, end; 1596 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1597 1598 if (!irq_ptr) 1599 return -ENODEV; 1600 q = irq_ptr->input_qs[nr]; 1601 WARN_ON(queue_irqs_enabled(q)); 1602 1603 qdio_sync_after_thinint(q); 1604 1605 /* 1606 * The interrupt could be caused by a PCI request. Check the 1607 * PCI capable outbound queues. 1608 */ 1609 qdio_check_outbound_after_thinint(q); 1610 1611 if (!qdio_inbound_q_moved(q)) 1612 return 0; 1613 1614 /* Note: upper-layer MUST stop processing immediately here ... */ 1615 if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)) 1616 return -EIO; 1617 1618 start = q->first_to_kick; 1619 end = q->first_to_check; 1620 *bufnr = start; 1621 *error = q->qdio_error; 1622 1623 /* for the next time */ 1624 q->first_to_kick = end; 1625 q->qdio_error = 0; 1626 return sub_buf(end, start); 1627 } 1628 EXPORT_SYMBOL(qdio_get_next_buffers); 1629 1630 /** 1631 * qdio_stop_irq - disable interrupt processing for the device 1632 * @cdev: associated ccw_device for the qdio subchannel 1633 * @nr: input queue number 1634 * 1635 * Return codes 1636 * 0 - interrupts were already disabled 1637 * 1 - interrupts successfully disabled 1638 */ 1639 int qdio_stop_irq(struct ccw_device *cdev, int nr) 1640 { 1641 struct qdio_q *q; 1642 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1643 1644 if (!irq_ptr) 1645 return -ENODEV; 1646 q = irq_ptr->input_qs[nr]; 1647 1648 if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED, 1649 &q->u.in.queue_irq_state)) 1650 return 0; 1651 else 1652 return 1; 1653 } 1654 EXPORT_SYMBOL(qdio_stop_irq); 1655 1656 static int __init init_QDIO(void) 1657 { 1658 int rc; 1659 1660 rc = qdio_setup_init(); 1661 if (rc) 1662 return rc; 1663 rc = tiqdio_allocate_memory(); 1664 if (rc) 1665 goto out_cache; 1666 rc = qdio_debug_init(); 1667 if (rc) 1668 goto out_ti; 1669 rc = tiqdio_register_thinints(); 1670 if (rc) 1671 goto out_debug; 1672 return 0; 1673 1674 out_debug: 1675 qdio_debug_exit(); 1676 out_ti: 1677 tiqdio_free_memory(); 1678 out_cache: 1679 qdio_setup_exit(); 1680 return rc; 1681 } 1682 1683 static void __exit exit_QDIO(void) 1684 { 1685 tiqdio_unregister_thinints(); 1686 tiqdio_free_memory(); 1687 qdio_debug_exit(); 1688 qdio_setup_exit(); 1689 } 1690 1691 module_init(init_QDIO); 1692 module_exit(exit_QDIO); 1693