1 /* Copyright (c) 2012 Coraid, Inc. See COPYING for GPL terms. */ 2 /* 3 * aoecmd.c 4 * Filesystem request handling methods 5 */ 6 7 #include <linux/ata.h> 8 #include <linux/slab.h> 9 #include <linux/hdreg.h> 10 #include <linux/blkdev.h> 11 #include <linux/skbuff.h> 12 #include <linux/netdevice.h> 13 #include <linux/genhd.h> 14 #include <linux/moduleparam.h> 15 #include <linux/workqueue.h> 16 #include <linux/kthread.h> 17 #include <net/net_namespace.h> 18 #include <asm/unaligned.h> 19 #include <linux/uio.h> 20 #include "aoe.h" 21 22 #define MAXIOC (8192) /* default meant to avoid most soft lockups */ 23 24 static void ktcomplete(struct frame *, struct sk_buff *); 25 static int count_targets(struct aoedev *d, int *untainted); 26 27 static struct buf *nextbuf(struct aoedev *); 28 29 static int aoe_deadsecs = 60 * 3; 30 module_param(aoe_deadsecs, int, 0644); 31 MODULE_PARM_DESC(aoe_deadsecs, "After aoe_deadsecs seconds, give up and fail dev."); 32 33 static int aoe_maxout = 64; 34 module_param(aoe_maxout, int, 0644); 35 MODULE_PARM_DESC(aoe_maxout, 36 "Only aoe_maxout outstanding packets for every MAC on eX.Y."); 37 38 static wait_queue_head_t ktiowq; 39 static struct ktstate kts; 40 41 /* io completion queue */ 42 static struct { 43 struct list_head head; 44 spinlock_t lock; 45 } iocq; 46 47 static struct page *empty_page; 48 49 static struct sk_buff * 50 new_skb(ulong len) 51 { 52 struct sk_buff *skb; 53 54 skb = alloc_skb(len, GFP_ATOMIC); 55 if (skb) { 56 skb_reset_mac_header(skb); 57 skb_reset_network_header(skb); 58 skb->protocol = __constant_htons(ETH_P_AOE); 59 skb_checksum_none_assert(skb); 60 } 61 return skb; 62 } 63 64 static struct frame * 65 getframe_deferred(struct aoedev *d, u32 tag) 66 { 67 struct list_head *head, *pos, *nx; 68 struct frame *f; 69 70 head = &d->rexmitq; 71 list_for_each_safe(pos, nx, head) { 72 f = list_entry(pos, struct frame, head); 73 if (f->tag == tag) { 74 list_del(pos); 75 return f; 76 } 77 } 78 return NULL; 79 } 80 81 static struct frame * 82 getframe(struct aoedev *d, u32 tag) 83 { 84 struct frame *f; 85 struct list_head *head, *pos, *nx; 86 u32 n; 87 88 n = tag % NFACTIVE; 89 head = &d->factive[n]; 90 list_for_each_safe(pos, nx, head) { 91 f = list_entry(pos, struct frame, head); 92 if (f->tag == tag) { 93 list_del(pos); 94 return f; 95 } 96 } 97 return NULL; 98 } 99 100 /* 101 * Leave the top bit clear so we have tagspace for userland. 102 * The bottom 16 bits are the xmit tick for rexmit/rttavg processing. 103 * This driver reserves tag -1 to mean "unused frame." 104 */ 105 static int 106 newtag(struct aoedev *d) 107 { 108 register ulong n; 109 110 n = jiffies & 0xffff; 111 return n |= (++d->lasttag & 0x7fff) << 16; 112 } 113 114 static u32 115 aoehdr_atainit(struct aoedev *d, struct aoetgt *t, struct aoe_hdr *h) 116 { 117 u32 host_tag = newtag(d); 118 119 memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src); 120 memcpy(h->dst, t->addr, sizeof h->dst); 121 h->type = __constant_cpu_to_be16(ETH_P_AOE); 122 h->verfl = AOE_HVER; 123 h->major = cpu_to_be16(d->aoemajor); 124 h->minor = d->aoeminor; 125 h->cmd = AOECMD_ATA; 126 h->tag = cpu_to_be32(host_tag); 127 128 return host_tag; 129 } 130 131 static inline void 132 put_lba(struct aoe_atahdr *ah, sector_t lba) 133 { 134 ah->lba0 = lba; 135 ah->lba1 = lba >>= 8; 136 ah->lba2 = lba >>= 8; 137 ah->lba3 = lba >>= 8; 138 ah->lba4 = lba >>= 8; 139 ah->lba5 = lba >>= 8; 140 } 141 142 static struct aoeif * 143 ifrotate(struct aoetgt *t) 144 { 145 struct aoeif *ifp; 146 147 ifp = t->ifp; 148 ifp++; 149 if (ifp >= &t->ifs[NAOEIFS] || ifp->nd == NULL) 150 ifp = t->ifs; 151 if (ifp->nd == NULL) 152 return NULL; 153 return t->ifp = ifp; 154 } 155 156 static void 157 skb_pool_put(struct aoedev *d, struct sk_buff *skb) 158 { 159 __skb_queue_tail(&d->skbpool, skb); 160 } 161 162 static struct sk_buff * 163 skb_pool_get(struct aoedev *d) 164 { 165 struct sk_buff *skb = skb_peek(&d->skbpool); 166 167 if (skb && atomic_read(&skb_shinfo(skb)->dataref) == 1) { 168 __skb_unlink(skb, &d->skbpool); 169 return skb; 170 } 171 if (skb_queue_len(&d->skbpool) < NSKBPOOLMAX && 172 (skb = new_skb(ETH_ZLEN))) 173 return skb; 174 175 return NULL; 176 } 177 178 void 179 aoe_freetframe(struct frame *f) 180 { 181 struct aoetgt *t; 182 183 t = f->t; 184 f->buf = NULL; 185 f->lba = 0; 186 f->bv = NULL; 187 f->r_skb = NULL; 188 f->flags = 0; 189 list_add(&f->head, &t->ffree); 190 } 191 192 static struct frame * 193 newtframe(struct aoedev *d, struct aoetgt *t) 194 { 195 struct frame *f; 196 struct sk_buff *skb; 197 struct list_head *pos; 198 199 if (list_empty(&t->ffree)) { 200 if (t->falloc >= NSKBPOOLMAX*2) 201 return NULL; 202 f = kcalloc(1, sizeof(*f), GFP_ATOMIC); 203 if (f == NULL) 204 return NULL; 205 t->falloc++; 206 f->t = t; 207 } else { 208 pos = t->ffree.next; 209 list_del(pos); 210 f = list_entry(pos, struct frame, head); 211 } 212 213 skb = f->skb; 214 if (skb == NULL) { 215 f->skb = skb = new_skb(ETH_ZLEN); 216 if (!skb) { 217 bail: aoe_freetframe(f); 218 return NULL; 219 } 220 } 221 222 if (atomic_read(&skb_shinfo(skb)->dataref) != 1) { 223 skb = skb_pool_get(d); 224 if (skb == NULL) 225 goto bail; 226 skb_pool_put(d, f->skb); 227 f->skb = skb; 228 } 229 230 skb->truesize -= skb->data_len; 231 skb_shinfo(skb)->nr_frags = skb->data_len = 0; 232 skb_trim(skb, 0); 233 return f; 234 } 235 236 static struct frame * 237 newframe(struct aoedev *d) 238 { 239 struct frame *f; 240 struct aoetgt *t, **tt; 241 int totout = 0; 242 int use_tainted; 243 int has_untainted; 244 245 if (!d->targets || !d->targets[0]) { 246 printk(KERN_ERR "aoe: NULL TARGETS!\n"); 247 return NULL; 248 } 249 tt = d->tgt; /* last used target */ 250 for (use_tainted = 0, has_untainted = 0;;) { 251 tt++; 252 if (tt >= &d->targets[d->ntargets] || !*tt) 253 tt = d->targets; 254 t = *tt; 255 if (!t->taint) { 256 has_untainted = 1; 257 totout += t->nout; 258 } 259 if (t->nout < t->maxout 260 && (use_tainted || !t->taint) 261 && t->ifp->nd) { 262 f = newtframe(d, t); 263 if (f) { 264 ifrotate(t); 265 d->tgt = tt; 266 return f; 267 } 268 } 269 if (tt == d->tgt) { /* we've looped and found nada */ 270 if (!use_tainted && !has_untainted) 271 use_tainted = 1; 272 else 273 break; 274 } 275 } 276 if (totout == 0) { 277 d->kicked++; 278 d->flags |= DEVFL_KICKME; 279 } 280 return NULL; 281 } 282 283 static void 284 skb_fillup(struct sk_buff *skb, struct bio_vec *bv, ulong off, ulong cnt) 285 { 286 int frag = 0; 287 ulong fcnt; 288 loop: 289 fcnt = bv->bv_len - (off - bv->bv_offset); 290 if (fcnt > cnt) 291 fcnt = cnt; 292 skb_fill_page_desc(skb, frag++, bv->bv_page, off, fcnt); 293 cnt -= fcnt; 294 if (cnt <= 0) 295 return; 296 bv++; 297 off = bv->bv_offset; 298 goto loop; 299 } 300 301 static void 302 fhash(struct frame *f) 303 { 304 struct aoedev *d = f->t->d; 305 u32 n; 306 307 n = f->tag % NFACTIVE; 308 list_add_tail(&f->head, &d->factive[n]); 309 } 310 311 static void 312 ata_rw_frameinit(struct frame *f) 313 { 314 struct aoetgt *t; 315 struct aoe_hdr *h; 316 struct aoe_atahdr *ah; 317 struct sk_buff *skb; 318 char writebit, extbit; 319 320 skb = f->skb; 321 h = (struct aoe_hdr *) skb_mac_header(skb); 322 ah = (struct aoe_atahdr *) (h + 1); 323 skb_put(skb, sizeof(*h) + sizeof(*ah)); 324 memset(h, 0, skb->len); 325 326 writebit = 0x10; 327 extbit = 0x4; 328 329 t = f->t; 330 f->tag = aoehdr_atainit(t->d, t, h); 331 fhash(f); 332 t->nout++; 333 f->waited = 0; 334 f->waited_total = 0; 335 if (f->buf) 336 f->lba = f->buf->sector; 337 338 /* set up ata header */ 339 ah->scnt = f->bcnt >> 9; 340 put_lba(ah, f->lba); 341 if (t->d->flags & DEVFL_EXT) { 342 ah->aflags |= AOEAFL_EXT; 343 } else { 344 extbit = 0; 345 ah->lba3 &= 0x0f; 346 ah->lba3 |= 0xe0; /* LBA bit + obsolete 0xa0 */ 347 } 348 if (f->buf && bio_data_dir(f->buf->bio) == WRITE) { 349 skb_fillup(skb, f->bv, f->bv_off, f->bcnt); 350 ah->aflags |= AOEAFL_WRITE; 351 skb->len += f->bcnt; 352 skb->data_len = f->bcnt; 353 skb->truesize += f->bcnt; 354 t->wpkts++; 355 } else { 356 t->rpkts++; 357 writebit = 0; 358 } 359 360 ah->cmdstat = ATA_CMD_PIO_READ | writebit | extbit; 361 skb->dev = t->ifp->nd; 362 } 363 364 static int 365 aoecmd_ata_rw(struct aoedev *d) 366 { 367 struct frame *f; 368 struct buf *buf; 369 struct aoetgt *t; 370 struct sk_buff *skb; 371 struct sk_buff_head queue; 372 ulong bcnt, fbcnt; 373 374 buf = nextbuf(d); 375 if (buf == NULL) 376 return 0; 377 f = newframe(d); 378 if (f == NULL) 379 return 0; 380 t = *d->tgt; 381 bcnt = d->maxbcnt; 382 if (bcnt == 0) 383 bcnt = DEFAULTBCNT; 384 if (bcnt > buf->resid) 385 bcnt = buf->resid; 386 fbcnt = bcnt; 387 f->bv = buf->bv; 388 f->bv_off = f->bv->bv_offset + (f->bv->bv_len - buf->bv_resid); 389 do { 390 if (fbcnt < buf->bv_resid) { 391 buf->bv_resid -= fbcnt; 392 buf->resid -= fbcnt; 393 break; 394 } 395 fbcnt -= buf->bv_resid; 396 buf->resid -= buf->bv_resid; 397 if (buf->resid == 0) { 398 d->ip.buf = NULL; 399 break; 400 } 401 buf->bv++; 402 buf->bv_resid = buf->bv->bv_len; 403 WARN_ON(buf->bv_resid == 0); 404 } while (fbcnt); 405 406 /* initialize the headers & frame */ 407 f->buf = buf; 408 f->bcnt = bcnt; 409 ata_rw_frameinit(f); 410 411 /* mark all tracking fields and load out */ 412 buf->nframesout += 1; 413 buf->sector += bcnt >> 9; 414 415 skb = skb_clone(f->skb, GFP_ATOMIC); 416 if (skb) { 417 do_gettimeofday(&f->sent); 418 f->sent_jiffs = (u32) jiffies; 419 __skb_queue_head_init(&queue); 420 __skb_queue_tail(&queue, skb); 421 aoenet_xmit(&queue); 422 } 423 return 1; 424 } 425 426 /* some callers cannot sleep, and they can call this function, 427 * transmitting the packets later, when interrupts are on 428 */ 429 static void 430 aoecmd_cfg_pkts(ushort aoemajor, unsigned char aoeminor, struct sk_buff_head *queue) 431 { 432 struct aoe_hdr *h; 433 struct aoe_cfghdr *ch; 434 struct sk_buff *skb; 435 struct net_device *ifp; 436 437 rcu_read_lock(); 438 for_each_netdev_rcu(&init_net, ifp) { 439 dev_hold(ifp); 440 if (!is_aoe_netif(ifp)) 441 goto cont; 442 443 skb = new_skb(sizeof *h + sizeof *ch); 444 if (skb == NULL) { 445 printk(KERN_INFO "aoe: skb alloc failure\n"); 446 goto cont; 447 } 448 skb_put(skb, sizeof *h + sizeof *ch); 449 skb->dev = ifp; 450 __skb_queue_tail(queue, skb); 451 h = (struct aoe_hdr *) skb_mac_header(skb); 452 memset(h, 0, sizeof *h + sizeof *ch); 453 454 memset(h->dst, 0xff, sizeof h->dst); 455 memcpy(h->src, ifp->dev_addr, sizeof h->src); 456 h->type = __constant_cpu_to_be16(ETH_P_AOE); 457 h->verfl = AOE_HVER; 458 h->major = cpu_to_be16(aoemajor); 459 h->minor = aoeminor; 460 h->cmd = AOECMD_CFG; 461 462 cont: 463 dev_put(ifp); 464 } 465 rcu_read_unlock(); 466 } 467 468 static void 469 resend(struct aoedev *d, struct frame *f) 470 { 471 struct sk_buff *skb; 472 struct sk_buff_head queue; 473 struct aoe_hdr *h; 474 struct aoe_atahdr *ah; 475 struct aoetgt *t; 476 char buf[128]; 477 u32 n; 478 479 t = f->t; 480 n = newtag(d); 481 skb = f->skb; 482 if (ifrotate(t) == NULL) { 483 /* probably can't happen, but set it up to fail anyway */ 484 pr_info("aoe: resend: no interfaces to rotate to.\n"); 485 ktcomplete(f, NULL); 486 return; 487 } 488 h = (struct aoe_hdr *) skb_mac_header(skb); 489 ah = (struct aoe_atahdr *) (h+1); 490 491 if (!(f->flags & FFL_PROBE)) { 492 snprintf(buf, sizeof(buf), 493 "%15s e%ld.%d oldtag=%08x@%08lx newtag=%08x s=%pm d=%pm nout=%d\n", 494 "retransmit", d->aoemajor, d->aoeminor, 495 f->tag, jiffies, n, 496 h->src, h->dst, t->nout); 497 aoechr_error(buf); 498 } 499 500 f->tag = n; 501 fhash(f); 502 h->tag = cpu_to_be32(n); 503 memcpy(h->dst, t->addr, sizeof h->dst); 504 memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src); 505 506 skb->dev = t->ifp->nd; 507 skb = skb_clone(skb, GFP_ATOMIC); 508 if (skb == NULL) 509 return; 510 do_gettimeofday(&f->sent); 511 f->sent_jiffs = (u32) jiffies; 512 __skb_queue_head_init(&queue); 513 __skb_queue_tail(&queue, skb); 514 aoenet_xmit(&queue); 515 } 516 517 static int 518 tsince_hr(struct frame *f) 519 { 520 struct timeval now; 521 int n; 522 523 do_gettimeofday(&now); 524 n = now.tv_usec - f->sent.tv_usec; 525 n += (now.tv_sec - f->sent.tv_sec) * USEC_PER_SEC; 526 527 if (n < 0) 528 n = -n; 529 530 /* For relatively long periods, use jiffies to avoid 531 * discrepancies caused by updates to the system time. 532 * 533 * On system with HZ of 1000, 32-bits is over 49 days 534 * worth of jiffies, or over 71 minutes worth of usecs. 535 * 536 * Jiffies overflow is handled by subtraction of unsigned ints: 537 * (gdb) print (unsigned) 2 - (unsigned) 0xfffffffe 538 * $3 = 4 539 * (gdb) 540 */ 541 if (n > USEC_PER_SEC / 4) { 542 n = ((u32) jiffies) - f->sent_jiffs; 543 n *= USEC_PER_SEC / HZ; 544 } 545 546 return n; 547 } 548 549 static int 550 tsince(u32 tag) 551 { 552 int n; 553 554 n = jiffies & 0xffff; 555 n -= tag & 0xffff; 556 if (n < 0) 557 n += 1<<16; 558 return jiffies_to_usecs(n + 1); 559 } 560 561 static struct aoeif * 562 getif(struct aoetgt *t, struct net_device *nd) 563 { 564 struct aoeif *p, *e; 565 566 p = t->ifs; 567 e = p + NAOEIFS; 568 for (; p < e; p++) 569 if (p->nd == nd) 570 return p; 571 return NULL; 572 } 573 574 static void 575 ejectif(struct aoetgt *t, struct aoeif *ifp) 576 { 577 struct aoeif *e; 578 struct net_device *nd; 579 ulong n; 580 581 nd = ifp->nd; 582 e = t->ifs + NAOEIFS - 1; 583 n = (e - ifp) * sizeof *ifp; 584 memmove(ifp, ifp+1, n); 585 e->nd = NULL; 586 dev_put(nd); 587 } 588 589 static struct frame * 590 reassign_frame(struct frame *f) 591 { 592 struct frame *nf; 593 struct sk_buff *skb; 594 595 nf = newframe(f->t->d); 596 if (!nf) 597 return NULL; 598 if (nf->t == f->t) { 599 aoe_freetframe(nf); 600 return NULL; 601 } 602 603 skb = nf->skb; 604 nf->skb = f->skb; 605 nf->buf = f->buf; 606 nf->bcnt = f->bcnt; 607 nf->lba = f->lba; 608 nf->bv = f->bv; 609 nf->bv_off = f->bv_off; 610 nf->waited = 0; 611 nf->waited_total = f->waited_total; 612 nf->sent = f->sent; 613 nf->sent_jiffs = f->sent_jiffs; 614 f->skb = skb; 615 616 return nf; 617 } 618 619 static void 620 probe(struct aoetgt *t) 621 { 622 struct aoedev *d; 623 struct frame *f; 624 struct sk_buff *skb; 625 struct sk_buff_head queue; 626 size_t n, m; 627 int frag; 628 629 d = t->d; 630 f = newtframe(d, t); 631 if (!f) { 632 pr_err("%s %pm for e%ld.%d: %s\n", 633 "aoe: cannot probe remote address", 634 t->addr, 635 (long) d->aoemajor, d->aoeminor, 636 "no frame available"); 637 return; 638 } 639 f->flags |= FFL_PROBE; 640 ifrotate(t); 641 f->bcnt = t->d->maxbcnt ? t->d->maxbcnt : DEFAULTBCNT; 642 ata_rw_frameinit(f); 643 skb = f->skb; 644 for (frag = 0, n = f->bcnt; n > 0; ++frag, n -= m) { 645 if (n < PAGE_SIZE) 646 m = n; 647 else 648 m = PAGE_SIZE; 649 skb_fill_page_desc(skb, frag, empty_page, 0, m); 650 } 651 skb->len += f->bcnt; 652 skb->data_len = f->bcnt; 653 skb->truesize += f->bcnt; 654 655 skb = skb_clone(f->skb, GFP_ATOMIC); 656 if (skb) { 657 do_gettimeofday(&f->sent); 658 f->sent_jiffs = (u32) jiffies; 659 __skb_queue_head_init(&queue); 660 __skb_queue_tail(&queue, skb); 661 aoenet_xmit(&queue); 662 } 663 } 664 665 static long 666 rto(struct aoedev *d) 667 { 668 long t; 669 670 t = 2 * d->rttavg >> RTTSCALE; 671 t += 8 * d->rttdev >> RTTDSCALE; 672 if (t == 0) 673 t = 1; 674 675 return t; 676 } 677 678 static void 679 rexmit_deferred(struct aoedev *d) 680 { 681 struct aoetgt *t; 682 struct frame *f; 683 struct frame *nf; 684 struct list_head *pos, *nx, *head; 685 int since; 686 int untainted; 687 688 count_targets(d, &untainted); 689 690 head = &d->rexmitq; 691 list_for_each_safe(pos, nx, head) { 692 f = list_entry(pos, struct frame, head); 693 t = f->t; 694 if (t->taint) { 695 if (!(f->flags & FFL_PROBE)) { 696 nf = reassign_frame(f); 697 if (nf) { 698 if (t->nout_probes == 0 699 && untainted > 0) { 700 probe(t); 701 t->nout_probes++; 702 } 703 list_replace(&f->head, &nf->head); 704 pos = &nf->head; 705 aoe_freetframe(f); 706 f = nf; 707 t = f->t; 708 } 709 } else if (untainted < 1) { 710 /* don't probe w/o other untainted aoetgts */ 711 goto stop_probe; 712 } else if (tsince_hr(f) < t->taint * rto(d)) { 713 /* reprobe slowly when taint is high */ 714 continue; 715 } 716 } else if (f->flags & FFL_PROBE) { 717 stop_probe: /* don't probe untainted aoetgts */ 718 list_del(pos); 719 aoe_freetframe(f); 720 /* leaving d->kicked, because this is routine */ 721 f->t->d->flags |= DEVFL_KICKME; 722 continue; 723 } 724 if (t->nout >= t->maxout) 725 continue; 726 list_del(pos); 727 t->nout++; 728 if (f->flags & FFL_PROBE) 729 t->nout_probes++; 730 since = tsince_hr(f); 731 f->waited += since; 732 f->waited_total += since; 733 resend(d, f); 734 } 735 } 736 737 /* An aoetgt accumulates demerits quickly, and successful 738 * probing redeems the aoetgt slowly. 739 */ 740 static void 741 scorn(struct aoetgt *t) 742 { 743 int n; 744 745 n = t->taint++; 746 t->taint += t->taint * 2; 747 if (n > t->taint) 748 t->taint = n; 749 if (t->taint > MAX_TAINT) 750 t->taint = MAX_TAINT; 751 } 752 753 static int 754 count_targets(struct aoedev *d, int *untainted) 755 { 756 int i, good; 757 758 for (i = good = 0; i < d->ntargets && d->targets[i]; ++i) 759 if (d->targets[i]->taint == 0) 760 good++; 761 762 if (untainted) 763 *untainted = good; 764 return i; 765 } 766 767 static void 768 rexmit_timer(ulong vp) 769 { 770 struct aoedev *d; 771 struct aoetgt *t; 772 struct aoeif *ifp; 773 struct frame *f; 774 struct list_head *head, *pos, *nx; 775 LIST_HEAD(flist); 776 register long timeout; 777 ulong flags, n; 778 int i; 779 int utgts; /* number of aoetgt descriptors (not slots) */ 780 int since; 781 782 d = (struct aoedev *) vp; 783 784 spin_lock_irqsave(&d->lock, flags); 785 786 /* timeout based on observed timings and variations */ 787 timeout = rto(d); 788 789 utgts = count_targets(d, NULL); 790 791 if (d->flags & DEVFL_TKILL) { 792 spin_unlock_irqrestore(&d->lock, flags); 793 return; 794 } 795 796 /* collect all frames to rexmit into flist */ 797 for (i = 0; i < NFACTIVE; i++) { 798 head = &d->factive[i]; 799 list_for_each_safe(pos, nx, head) { 800 f = list_entry(pos, struct frame, head); 801 if (tsince_hr(f) < timeout) 802 break; /* end of expired frames */ 803 /* move to flist for later processing */ 804 list_move_tail(pos, &flist); 805 } 806 } 807 808 /* process expired frames */ 809 while (!list_empty(&flist)) { 810 pos = flist.next; 811 f = list_entry(pos, struct frame, head); 812 since = tsince_hr(f); 813 n = f->waited_total + since; 814 n /= USEC_PER_SEC; 815 if (aoe_deadsecs 816 && n > aoe_deadsecs 817 && !(f->flags & FFL_PROBE)) { 818 /* Waited too long. Device failure. 819 * Hang all frames on first hash bucket for downdev 820 * to clean up. 821 */ 822 list_splice(&flist, &d->factive[0]); 823 aoedev_downdev(d); 824 goto out; 825 } 826 827 t = f->t; 828 n = f->waited + since; 829 n /= USEC_PER_SEC; 830 if (aoe_deadsecs && utgts > 0 831 && (n > aoe_deadsecs / utgts || n > HARD_SCORN_SECS)) 832 scorn(t); /* avoid this target */ 833 834 if (t->maxout != 1) { 835 t->ssthresh = t->maxout / 2; 836 t->maxout = 1; 837 } 838 839 if (f->flags & FFL_PROBE) { 840 t->nout_probes--; 841 } else { 842 ifp = getif(t, f->skb->dev); 843 if (ifp && ++ifp->lost > (t->nframes << 1) 844 && (ifp != t->ifs || t->ifs[1].nd)) { 845 ejectif(t, ifp); 846 ifp = NULL; 847 } 848 } 849 list_move_tail(pos, &d->rexmitq); 850 t->nout--; 851 } 852 rexmit_deferred(d); 853 854 out: 855 if ((d->flags & DEVFL_KICKME) && d->blkq) { 856 d->flags &= ~DEVFL_KICKME; 857 d->blkq->request_fn(d->blkq); 858 } 859 860 d->timer.expires = jiffies + TIMERTICK; 861 add_timer(&d->timer); 862 863 spin_unlock_irqrestore(&d->lock, flags); 864 } 865 866 static unsigned long 867 rqbiocnt(struct request *r) 868 { 869 struct bio *bio; 870 unsigned long n = 0; 871 872 __rq_for_each_bio(bio, r) 873 n++; 874 return n; 875 } 876 877 /* This can be removed if we are certain that no users of the block 878 * layer will ever use zero-count pages in bios. Otherwise we have to 879 * protect against the put_page sometimes done by the network layer. 880 * 881 * See http://oss.sgi.com/archives/xfs/2007-01/msg00594.html for 882 * discussion. 883 * 884 * We cannot use get_page in the workaround, because it insists on a 885 * positive page count as a precondition. So we use _count directly. 886 */ 887 static void 888 bio_pageinc(struct bio *bio) 889 { 890 struct bio_vec *bv; 891 struct page *page; 892 int i; 893 894 bio_for_each_segment(bv, bio, i) { 895 page = bv->bv_page; 896 /* Non-zero page count for non-head members of 897 * compound pages is no longer allowed by the kernel, 898 * but this has never been seen here. 899 */ 900 if (unlikely(PageCompound(page))) 901 if (compound_trans_head(page) != page) { 902 pr_crit("page tail used for block I/O\n"); 903 BUG(); 904 } 905 atomic_inc(&page->_count); 906 } 907 } 908 909 static void 910 bio_pagedec(struct bio *bio) 911 { 912 struct bio_vec *bv; 913 int i; 914 915 bio_for_each_segment(bv, bio, i) 916 atomic_dec(&bv->bv_page->_count); 917 } 918 919 static void 920 bufinit(struct buf *buf, struct request *rq, struct bio *bio) 921 { 922 struct bio_vec *bv; 923 924 memset(buf, 0, sizeof(*buf)); 925 buf->rq = rq; 926 buf->bio = bio; 927 buf->resid = bio->bi_size; 928 buf->sector = bio->bi_sector; 929 bio_pageinc(bio); 930 buf->bv = bv = &bio->bi_io_vec[bio->bi_idx]; 931 buf->bv_resid = bv->bv_len; 932 WARN_ON(buf->bv_resid == 0); 933 } 934 935 static struct buf * 936 nextbuf(struct aoedev *d) 937 { 938 struct request *rq; 939 struct request_queue *q; 940 struct buf *buf; 941 struct bio *bio; 942 943 q = d->blkq; 944 if (q == NULL) 945 return NULL; /* initializing */ 946 if (d->ip.buf) 947 return d->ip.buf; 948 rq = d->ip.rq; 949 if (rq == NULL) { 950 rq = blk_peek_request(q); 951 if (rq == NULL) 952 return NULL; 953 blk_start_request(rq); 954 d->ip.rq = rq; 955 d->ip.nxbio = rq->bio; 956 rq->special = (void *) rqbiocnt(rq); 957 } 958 buf = mempool_alloc(d->bufpool, GFP_ATOMIC); 959 if (buf == NULL) { 960 pr_err("aoe: nextbuf: unable to mempool_alloc!\n"); 961 return NULL; 962 } 963 bio = d->ip.nxbio; 964 bufinit(buf, rq, bio); 965 bio = bio->bi_next; 966 d->ip.nxbio = bio; 967 if (bio == NULL) 968 d->ip.rq = NULL; 969 return d->ip.buf = buf; 970 } 971 972 /* enters with d->lock held */ 973 void 974 aoecmd_work(struct aoedev *d) 975 { 976 rexmit_deferred(d); 977 while (aoecmd_ata_rw(d)) 978 ; 979 } 980 981 /* this function performs work that has been deferred until sleeping is OK 982 */ 983 void 984 aoecmd_sleepwork(struct work_struct *work) 985 { 986 struct aoedev *d = container_of(work, struct aoedev, work); 987 struct block_device *bd; 988 u64 ssize; 989 990 if (d->flags & DEVFL_GDALLOC) 991 aoeblk_gdalloc(d); 992 993 if (d->flags & DEVFL_NEWSIZE) { 994 ssize = get_capacity(d->gd); 995 bd = bdget_disk(d->gd, 0); 996 if (bd) { 997 mutex_lock(&bd->bd_inode->i_mutex); 998 i_size_write(bd->bd_inode, (loff_t)ssize<<9); 999 mutex_unlock(&bd->bd_inode->i_mutex); 1000 bdput(bd); 1001 } 1002 spin_lock_irq(&d->lock); 1003 d->flags |= DEVFL_UP; 1004 d->flags &= ~DEVFL_NEWSIZE; 1005 spin_unlock_irq(&d->lock); 1006 } 1007 } 1008 1009 static void 1010 ata_ident_fixstring(u16 *id, int ns) 1011 { 1012 u16 s; 1013 1014 while (ns-- > 0) { 1015 s = *id; 1016 *id++ = s >> 8 | s << 8; 1017 } 1018 } 1019 1020 static void 1021 ataid_complete(struct aoedev *d, struct aoetgt *t, unsigned char *id) 1022 { 1023 u64 ssize; 1024 u16 n; 1025 1026 /* word 83: command set supported */ 1027 n = get_unaligned_le16(&id[83 << 1]); 1028 1029 /* word 86: command set/feature enabled */ 1030 n |= get_unaligned_le16(&id[86 << 1]); 1031 1032 if (n & (1<<10)) { /* bit 10: LBA 48 */ 1033 d->flags |= DEVFL_EXT; 1034 1035 /* word 100: number lba48 sectors */ 1036 ssize = get_unaligned_le64(&id[100 << 1]); 1037 1038 /* set as in ide-disk.c:init_idedisk_capacity */ 1039 d->geo.cylinders = ssize; 1040 d->geo.cylinders /= (255 * 63); 1041 d->geo.heads = 255; 1042 d->geo.sectors = 63; 1043 } else { 1044 d->flags &= ~DEVFL_EXT; 1045 1046 /* number lba28 sectors */ 1047 ssize = get_unaligned_le32(&id[60 << 1]); 1048 1049 /* NOTE: obsolete in ATA 6 */ 1050 d->geo.cylinders = get_unaligned_le16(&id[54 << 1]); 1051 d->geo.heads = get_unaligned_le16(&id[55 << 1]); 1052 d->geo.sectors = get_unaligned_le16(&id[56 << 1]); 1053 } 1054 1055 ata_ident_fixstring((u16 *) &id[10<<1], 10); /* serial */ 1056 ata_ident_fixstring((u16 *) &id[23<<1], 4); /* firmware */ 1057 ata_ident_fixstring((u16 *) &id[27<<1], 20); /* model */ 1058 memcpy(d->ident, id, sizeof(d->ident)); 1059 1060 if (d->ssize != ssize) 1061 printk(KERN_INFO 1062 "aoe: %pm e%ld.%d v%04x has %llu sectors\n", 1063 t->addr, 1064 d->aoemajor, d->aoeminor, 1065 d->fw_ver, (long long)ssize); 1066 d->ssize = ssize; 1067 d->geo.start = 0; 1068 if (d->flags & (DEVFL_GDALLOC|DEVFL_NEWSIZE)) 1069 return; 1070 if (d->gd != NULL) { 1071 set_capacity(d->gd, ssize); 1072 d->flags |= DEVFL_NEWSIZE; 1073 } else 1074 d->flags |= DEVFL_GDALLOC; 1075 schedule_work(&d->work); 1076 } 1077 1078 static void 1079 calc_rttavg(struct aoedev *d, struct aoetgt *t, int rtt) 1080 { 1081 register long n; 1082 1083 n = rtt; 1084 1085 /* cf. Congestion Avoidance and Control, Jacobson & Karels, 1988 */ 1086 n -= d->rttavg >> RTTSCALE; 1087 d->rttavg += n; 1088 if (n < 0) 1089 n = -n; 1090 n -= d->rttdev >> RTTDSCALE; 1091 d->rttdev += n; 1092 1093 if (!t || t->maxout >= t->nframes) 1094 return; 1095 if (t->maxout < t->ssthresh) 1096 t->maxout += 1; 1097 else if (t->nout == t->maxout && t->next_cwnd-- == 0) { 1098 t->maxout += 1; 1099 t->next_cwnd = t->maxout; 1100 } 1101 } 1102 1103 static struct aoetgt * 1104 gettgt(struct aoedev *d, char *addr) 1105 { 1106 struct aoetgt **t, **e; 1107 1108 t = d->targets; 1109 e = t + d->ntargets; 1110 for (; t < e && *t; t++) 1111 if (memcmp((*t)->addr, addr, sizeof((*t)->addr)) == 0) 1112 return *t; 1113 return NULL; 1114 } 1115 1116 static void 1117 bvcpy(struct bio_vec *bv, ulong off, struct sk_buff *skb, long cnt) 1118 { 1119 ulong fcnt; 1120 char *p; 1121 int soff = 0; 1122 loop: 1123 fcnt = bv->bv_len - (off - bv->bv_offset); 1124 if (fcnt > cnt) 1125 fcnt = cnt; 1126 p = page_address(bv->bv_page) + off; 1127 skb_copy_bits(skb, soff, p, fcnt); 1128 soff += fcnt; 1129 cnt -= fcnt; 1130 if (cnt <= 0) 1131 return; 1132 bv++; 1133 off = bv->bv_offset; 1134 goto loop; 1135 } 1136 1137 void 1138 aoe_end_request(struct aoedev *d, struct request *rq, int fastfail) 1139 { 1140 struct bio *bio; 1141 int bok; 1142 struct request_queue *q; 1143 1144 q = d->blkq; 1145 if (rq == d->ip.rq) 1146 d->ip.rq = NULL; 1147 do { 1148 bio = rq->bio; 1149 bok = !fastfail && test_bit(BIO_UPTODATE, &bio->bi_flags); 1150 } while (__blk_end_request(rq, bok ? 0 : -EIO, bio->bi_size)); 1151 1152 /* cf. http://lkml.org/lkml/2006/10/31/28 */ 1153 if (!fastfail) 1154 __blk_run_queue(q); 1155 } 1156 1157 static void 1158 aoe_end_buf(struct aoedev *d, struct buf *buf) 1159 { 1160 struct request *rq; 1161 unsigned long n; 1162 1163 if (buf == d->ip.buf) 1164 d->ip.buf = NULL; 1165 rq = buf->rq; 1166 bio_pagedec(buf->bio); 1167 mempool_free(buf, d->bufpool); 1168 n = (unsigned long) rq->special; 1169 rq->special = (void *) --n; 1170 if (n == 0) 1171 aoe_end_request(d, rq, 0); 1172 } 1173 1174 static void 1175 ktiocomplete(struct frame *f) 1176 { 1177 struct aoe_hdr *hin, *hout; 1178 struct aoe_atahdr *ahin, *ahout; 1179 struct buf *buf; 1180 struct sk_buff *skb; 1181 struct aoetgt *t; 1182 struct aoeif *ifp; 1183 struct aoedev *d; 1184 long n; 1185 int untainted; 1186 1187 if (f == NULL) 1188 return; 1189 1190 t = f->t; 1191 d = t->d; 1192 skb = f->r_skb; 1193 buf = f->buf; 1194 if (f->flags & FFL_PROBE) 1195 goto out; 1196 if (!skb) /* just fail the buf. */ 1197 goto noskb; 1198 1199 hout = (struct aoe_hdr *) skb_mac_header(f->skb); 1200 ahout = (struct aoe_atahdr *) (hout+1); 1201 1202 hin = (struct aoe_hdr *) skb->data; 1203 skb_pull(skb, sizeof(*hin)); 1204 ahin = (struct aoe_atahdr *) skb->data; 1205 skb_pull(skb, sizeof(*ahin)); 1206 if (ahin->cmdstat & 0xa9) { /* these bits cleared on success */ 1207 pr_err("aoe: ata error cmd=%2.2Xh stat=%2.2Xh from e%ld.%d\n", 1208 ahout->cmdstat, ahin->cmdstat, 1209 d->aoemajor, d->aoeminor); 1210 noskb: if (buf) 1211 clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); 1212 goto out; 1213 } 1214 1215 n = ahout->scnt << 9; 1216 switch (ahout->cmdstat) { 1217 case ATA_CMD_PIO_READ: 1218 case ATA_CMD_PIO_READ_EXT: 1219 if (skb->len < n) { 1220 pr_err("%s e%ld.%d. skb->len=%d need=%ld\n", 1221 "aoe: runt data size in read from", 1222 (long) d->aoemajor, d->aoeminor, 1223 skb->len, n); 1224 clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); 1225 break; 1226 } 1227 bvcpy(f->bv, f->bv_off, skb, n); 1228 case ATA_CMD_PIO_WRITE: 1229 case ATA_CMD_PIO_WRITE_EXT: 1230 spin_lock_irq(&d->lock); 1231 ifp = getif(t, skb->dev); 1232 if (ifp) 1233 ifp->lost = 0; 1234 spin_unlock_irq(&d->lock); 1235 break; 1236 case ATA_CMD_ID_ATA: 1237 if (skb->len < 512) { 1238 pr_info("%s e%ld.%d. skb->len=%d need=512\n", 1239 "aoe: runt data size in ataid from", 1240 (long) d->aoemajor, d->aoeminor, 1241 skb->len); 1242 break; 1243 } 1244 if (skb_linearize(skb)) 1245 break; 1246 spin_lock_irq(&d->lock); 1247 ataid_complete(d, t, skb->data); 1248 spin_unlock_irq(&d->lock); 1249 break; 1250 default: 1251 pr_info("aoe: unrecognized ata command %2.2Xh for %d.%d\n", 1252 ahout->cmdstat, 1253 be16_to_cpu(get_unaligned(&hin->major)), 1254 hin->minor); 1255 } 1256 out: 1257 spin_lock_irq(&d->lock); 1258 if (t->taint > 0 1259 && --t->taint > 0 1260 && t->nout_probes == 0) { 1261 count_targets(d, &untainted); 1262 if (untainted > 0) { 1263 probe(t); 1264 t->nout_probes++; 1265 } 1266 } 1267 1268 aoe_freetframe(f); 1269 1270 if (buf && --buf->nframesout == 0 && buf->resid == 0) 1271 aoe_end_buf(d, buf); 1272 1273 spin_unlock_irq(&d->lock); 1274 aoedev_put(d); 1275 dev_kfree_skb(skb); 1276 } 1277 1278 /* Enters with iocq.lock held. 1279 * Returns true iff responses needing processing remain. 1280 */ 1281 static int 1282 ktio(void) 1283 { 1284 struct frame *f; 1285 struct list_head *pos; 1286 int i; 1287 1288 for (i = 0; ; ++i) { 1289 if (i == MAXIOC) 1290 return 1; 1291 if (list_empty(&iocq.head)) 1292 return 0; 1293 pos = iocq.head.next; 1294 list_del(pos); 1295 spin_unlock_irq(&iocq.lock); 1296 f = list_entry(pos, struct frame, head); 1297 ktiocomplete(f); 1298 spin_lock_irq(&iocq.lock); 1299 } 1300 } 1301 1302 static int 1303 kthread(void *vp) 1304 { 1305 struct ktstate *k; 1306 DECLARE_WAITQUEUE(wait, current); 1307 int more; 1308 1309 k = vp; 1310 current->flags |= PF_NOFREEZE; 1311 set_user_nice(current, -10); 1312 complete(&k->rendez); /* tell spawner we're running */ 1313 do { 1314 spin_lock_irq(k->lock); 1315 more = k->fn(); 1316 if (!more) { 1317 add_wait_queue(k->waitq, &wait); 1318 __set_current_state(TASK_INTERRUPTIBLE); 1319 } 1320 spin_unlock_irq(k->lock); 1321 if (!more) { 1322 schedule(); 1323 remove_wait_queue(k->waitq, &wait); 1324 } else 1325 cond_resched(); 1326 } while (!kthread_should_stop()); 1327 complete(&k->rendez); /* tell spawner we're stopping */ 1328 return 0; 1329 } 1330 1331 void 1332 aoe_ktstop(struct ktstate *k) 1333 { 1334 kthread_stop(k->task); 1335 wait_for_completion(&k->rendez); 1336 } 1337 1338 int 1339 aoe_ktstart(struct ktstate *k) 1340 { 1341 struct task_struct *task; 1342 1343 init_completion(&k->rendez); 1344 task = kthread_run(kthread, k, k->name); 1345 if (task == NULL || IS_ERR(task)) 1346 return -ENOMEM; 1347 k->task = task; 1348 wait_for_completion(&k->rendez); /* allow kthread to start */ 1349 init_completion(&k->rendez); /* for waiting for exit later */ 1350 return 0; 1351 } 1352 1353 /* pass it off to kthreads for processing */ 1354 static void 1355 ktcomplete(struct frame *f, struct sk_buff *skb) 1356 { 1357 ulong flags; 1358 1359 f->r_skb = skb; 1360 spin_lock_irqsave(&iocq.lock, flags); 1361 list_add_tail(&f->head, &iocq.head); 1362 spin_unlock_irqrestore(&iocq.lock, flags); 1363 wake_up(&ktiowq); 1364 } 1365 1366 struct sk_buff * 1367 aoecmd_ata_rsp(struct sk_buff *skb) 1368 { 1369 struct aoedev *d; 1370 struct aoe_hdr *h; 1371 struct frame *f; 1372 u32 n; 1373 ulong flags; 1374 char ebuf[128]; 1375 u16 aoemajor; 1376 1377 h = (struct aoe_hdr *) skb->data; 1378 aoemajor = be16_to_cpu(get_unaligned(&h->major)); 1379 d = aoedev_by_aoeaddr(aoemajor, h->minor, 0); 1380 if (d == NULL) { 1381 snprintf(ebuf, sizeof ebuf, "aoecmd_ata_rsp: ata response " 1382 "for unknown device %d.%d\n", 1383 aoemajor, h->minor); 1384 aoechr_error(ebuf); 1385 return skb; 1386 } 1387 1388 spin_lock_irqsave(&d->lock, flags); 1389 1390 n = be32_to_cpu(get_unaligned(&h->tag)); 1391 f = getframe(d, n); 1392 if (f) { 1393 calc_rttavg(d, f->t, tsince_hr(f)); 1394 f->t->nout--; 1395 if (f->flags & FFL_PROBE) 1396 f->t->nout_probes--; 1397 } else { 1398 f = getframe_deferred(d, n); 1399 if (f) { 1400 calc_rttavg(d, NULL, tsince_hr(f)); 1401 } else { 1402 calc_rttavg(d, NULL, tsince(n)); 1403 spin_unlock_irqrestore(&d->lock, flags); 1404 aoedev_put(d); 1405 snprintf(ebuf, sizeof(ebuf), 1406 "%15s e%d.%d tag=%08x@%08lx s=%pm d=%pm\n", 1407 "unexpected rsp", 1408 get_unaligned_be16(&h->major), 1409 h->minor, 1410 get_unaligned_be32(&h->tag), 1411 jiffies, 1412 h->src, 1413 h->dst); 1414 aoechr_error(ebuf); 1415 return skb; 1416 } 1417 } 1418 aoecmd_work(d); 1419 1420 spin_unlock_irqrestore(&d->lock, flags); 1421 1422 ktcomplete(f, skb); 1423 1424 /* 1425 * Note here that we do not perform an aoedev_put, as we are 1426 * leaving this reference for the ktio to release. 1427 */ 1428 return NULL; 1429 } 1430 1431 void 1432 aoecmd_cfg(ushort aoemajor, unsigned char aoeminor) 1433 { 1434 struct sk_buff_head queue; 1435 1436 __skb_queue_head_init(&queue); 1437 aoecmd_cfg_pkts(aoemajor, aoeminor, &queue); 1438 aoenet_xmit(&queue); 1439 } 1440 1441 struct sk_buff * 1442 aoecmd_ata_id(struct aoedev *d) 1443 { 1444 struct aoe_hdr *h; 1445 struct aoe_atahdr *ah; 1446 struct frame *f; 1447 struct sk_buff *skb; 1448 struct aoetgt *t; 1449 1450 f = newframe(d); 1451 if (f == NULL) 1452 return NULL; 1453 1454 t = *d->tgt; 1455 1456 /* initialize the headers & frame */ 1457 skb = f->skb; 1458 h = (struct aoe_hdr *) skb_mac_header(skb); 1459 ah = (struct aoe_atahdr *) (h+1); 1460 skb_put(skb, sizeof *h + sizeof *ah); 1461 memset(h, 0, skb->len); 1462 f->tag = aoehdr_atainit(d, t, h); 1463 fhash(f); 1464 t->nout++; 1465 f->waited = 0; 1466 f->waited_total = 0; 1467 1468 /* set up ata header */ 1469 ah->scnt = 1; 1470 ah->cmdstat = ATA_CMD_ID_ATA; 1471 ah->lba3 = 0xa0; 1472 1473 skb->dev = t->ifp->nd; 1474 1475 d->rttavg = RTTAVG_INIT; 1476 d->rttdev = RTTDEV_INIT; 1477 d->timer.function = rexmit_timer; 1478 1479 skb = skb_clone(skb, GFP_ATOMIC); 1480 if (skb) { 1481 do_gettimeofday(&f->sent); 1482 f->sent_jiffs = (u32) jiffies; 1483 } 1484 1485 return skb; 1486 } 1487 1488 static struct aoetgt ** 1489 grow_targets(struct aoedev *d) 1490 { 1491 ulong oldn, newn; 1492 struct aoetgt **tt; 1493 1494 oldn = d->ntargets; 1495 newn = oldn * 2; 1496 tt = kcalloc(newn, sizeof(*d->targets), GFP_ATOMIC); 1497 if (!tt) 1498 return NULL; 1499 memmove(tt, d->targets, sizeof(*d->targets) * oldn); 1500 d->tgt = tt + (d->tgt - d->targets); 1501 kfree(d->targets); 1502 d->targets = tt; 1503 d->ntargets = newn; 1504 1505 return &d->targets[oldn]; 1506 } 1507 1508 static struct aoetgt * 1509 addtgt(struct aoedev *d, char *addr, ulong nframes) 1510 { 1511 struct aoetgt *t, **tt, **te; 1512 1513 tt = d->targets; 1514 te = tt + d->ntargets; 1515 for (; tt < te && *tt; tt++) 1516 ; 1517 1518 if (tt == te) { 1519 tt = grow_targets(d); 1520 if (!tt) 1521 goto nomem; 1522 } 1523 t = kzalloc(sizeof(*t), GFP_ATOMIC); 1524 if (!t) 1525 goto nomem; 1526 t->nframes = nframes; 1527 t->d = d; 1528 memcpy(t->addr, addr, sizeof t->addr); 1529 t->ifp = t->ifs; 1530 aoecmd_wreset(t); 1531 t->maxout = t->nframes / 2; 1532 INIT_LIST_HEAD(&t->ffree); 1533 return *tt = t; 1534 1535 nomem: 1536 pr_info("aoe: cannot allocate memory to add target\n"); 1537 return NULL; 1538 } 1539 1540 static void 1541 setdbcnt(struct aoedev *d) 1542 { 1543 struct aoetgt **t, **e; 1544 int bcnt = 0; 1545 1546 t = d->targets; 1547 e = t + d->ntargets; 1548 for (; t < e && *t; t++) 1549 if (bcnt == 0 || bcnt > (*t)->minbcnt) 1550 bcnt = (*t)->minbcnt; 1551 if (bcnt != d->maxbcnt) { 1552 d->maxbcnt = bcnt; 1553 pr_info("aoe: e%ld.%d: setting %d byte data frames\n", 1554 d->aoemajor, d->aoeminor, bcnt); 1555 } 1556 } 1557 1558 static void 1559 setifbcnt(struct aoetgt *t, struct net_device *nd, int bcnt) 1560 { 1561 struct aoedev *d; 1562 struct aoeif *p, *e; 1563 int minbcnt; 1564 1565 d = t->d; 1566 minbcnt = bcnt; 1567 p = t->ifs; 1568 e = p + NAOEIFS; 1569 for (; p < e; p++) { 1570 if (p->nd == NULL) 1571 break; /* end of the valid interfaces */ 1572 if (p->nd == nd) { 1573 p->bcnt = bcnt; /* we're updating */ 1574 nd = NULL; 1575 } else if (minbcnt > p->bcnt) 1576 minbcnt = p->bcnt; /* find the min interface */ 1577 } 1578 if (nd) { 1579 if (p == e) { 1580 pr_err("aoe: device setifbcnt failure; too many interfaces.\n"); 1581 return; 1582 } 1583 dev_hold(nd); 1584 p->nd = nd; 1585 p->bcnt = bcnt; 1586 } 1587 t->minbcnt = minbcnt; 1588 setdbcnt(d); 1589 } 1590 1591 void 1592 aoecmd_cfg_rsp(struct sk_buff *skb) 1593 { 1594 struct aoedev *d; 1595 struct aoe_hdr *h; 1596 struct aoe_cfghdr *ch; 1597 struct aoetgt *t; 1598 ulong flags, aoemajor; 1599 struct sk_buff *sl; 1600 struct sk_buff_head queue; 1601 u16 n; 1602 1603 sl = NULL; 1604 h = (struct aoe_hdr *) skb_mac_header(skb); 1605 ch = (struct aoe_cfghdr *) (h+1); 1606 1607 /* 1608 * Enough people have their dip switches set backwards to 1609 * warrant a loud message for this special case. 1610 */ 1611 aoemajor = get_unaligned_be16(&h->major); 1612 if (aoemajor == 0xfff) { 1613 printk(KERN_ERR "aoe: Warning: shelf address is all ones. " 1614 "Check shelf dip switches.\n"); 1615 return; 1616 } 1617 if (aoemajor == 0xffff) { 1618 pr_info("aoe: e%ld.%d: broadcast shelf number invalid\n", 1619 aoemajor, (int) h->minor); 1620 return; 1621 } 1622 if (h->minor == 0xff) { 1623 pr_info("aoe: e%ld.%d: broadcast slot number invalid\n", 1624 aoemajor, (int) h->minor); 1625 return; 1626 } 1627 1628 n = be16_to_cpu(ch->bufcnt); 1629 if (n > aoe_maxout) /* keep it reasonable */ 1630 n = aoe_maxout; 1631 1632 d = aoedev_by_aoeaddr(aoemajor, h->minor, 1); 1633 if (d == NULL) { 1634 pr_info("aoe: device allocation failure\n"); 1635 return; 1636 } 1637 1638 spin_lock_irqsave(&d->lock, flags); 1639 1640 t = gettgt(d, h->src); 1641 if (t) { 1642 t->nframes = n; 1643 if (n < t->maxout) 1644 aoecmd_wreset(t); 1645 } else { 1646 t = addtgt(d, h->src, n); 1647 if (!t) 1648 goto bail; 1649 } 1650 n = skb->dev->mtu; 1651 n -= sizeof(struct aoe_hdr) + sizeof(struct aoe_atahdr); 1652 n /= 512; 1653 if (n > ch->scnt) 1654 n = ch->scnt; 1655 n = n ? n * 512 : DEFAULTBCNT; 1656 setifbcnt(t, skb->dev, n); 1657 1658 /* don't change users' perspective */ 1659 if (d->nopen == 0) { 1660 d->fw_ver = be16_to_cpu(ch->fwver); 1661 sl = aoecmd_ata_id(d); 1662 } 1663 bail: 1664 spin_unlock_irqrestore(&d->lock, flags); 1665 aoedev_put(d); 1666 if (sl) { 1667 __skb_queue_head_init(&queue); 1668 __skb_queue_tail(&queue, sl); 1669 aoenet_xmit(&queue); 1670 } 1671 } 1672 1673 void 1674 aoecmd_wreset(struct aoetgt *t) 1675 { 1676 t->maxout = 1; 1677 t->ssthresh = t->nframes / 2; 1678 t->next_cwnd = t->nframes; 1679 } 1680 1681 void 1682 aoecmd_cleanslate(struct aoedev *d) 1683 { 1684 struct aoetgt **t, **te; 1685 1686 d->rttavg = RTTAVG_INIT; 1687 d->rttdev = RTTDEV_INIT; 1688 d->maxbcnt = 0; 1689 1690 t = d->targets; 1691 te = t + d->ntargets; 1692 for (; t < te && *t; t++) 1693 aoecmd_wreset(*t); 1694 } 1695 1696 void 1697 aoe_failbuf(struct aoedev *d, struct buf *buf) 1698 { 1699 if (buf == NULL) 1700 return; 1701 buf->resid = 0; 1702 clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); 1703 if (buf->nframesout == 0) 1704 aoe_end_buf(d, buf); 1705 } 1706 1707 void 1708 aoe_flush_iocq(void) 1709 { 1710 struct frame *f; 1711 struct aoedev *d; 1712 LIST_HEAD(flist); 1713 struct list_head *pos; 1714 struct sk_buff *skb; 1715 ulong flags; 1716 1717 spin_lock_irqsave(&iocq.lock, flags); 1718 list_splice_init(&iocq.head, &flist); 1719 spin_unlock_irqrestore(&iocq.lock, flags); 1720 while (!list_empty(&flist)) { 1721 pos = flist.next; 1722 list_del(pos); 1723 f = list_entry(pos, struct frame, head); 1724 d = f->t->d; 1725 skb = f->r_skb; 1726 spin_lock_irqsave(&d->lock, flags); 1727 if (f->buf) { 1728 f->buf->nframesout--; 1729 aoe_failbuf(d, f->buf); 1730 } 1731 aoe_freetframe(f); 1732 spin_unlock_irqrestore(&d->lock, flags); 1733 dev_kfree_skb(skb); 1734 aoedev_put(d); 1735 } 1736 } 1737 1738 int __init 1739 aoecmd_init(void) 1740 { 1741 void *p; 1742 1743 /* get_zeroed_page returns page with ref count 1 */ 1744 p = (void *) get_zeroed_page(GFP_KERNEL | __GFP_REPEAT); 1745 if (!p) 1746 return -ENOMEM; 1747 empty_page = virt_to_page(p); 1748 1749 INIT_LIST_HEAD(&iocq.head); 1750 spin_lock_init(&iocq.lock); 1751 init_waitqueue_head(&ktiowq); 1752 kts.name = "aoe_ktio"; 1753 kts.fn = ktio; 1754 kts.waitq = &ktiowq; 1755 kts.lock = &iocq.lock; 1756 return aoe_ktstart(&kts); 1757 } 1758 1759 void 1760 aoecmd_exit(void) 1761 { 1762 aoe_ktstop(&kts); 1763 aoe_flush_iocq(); 1764 1765 free_page((unsigned long) page_address(empty_page)); 1766 empty_page = NULL; 1767 } 1768