1ca47bbd9SEd Cashin /* Copyright (c) 2013 Coraid, Inc. See COPYING for GPL terms. */
21da177e4SLinus Torvalds /*
31da177e4SLinus Torvalds * aoecmd.c
41da177e4SLinus Torvalds * Filesystem request handling methods
51da177e4SLinus Torvalds */
61da177e4SLinus Torvalds
704b3ab52SBartlomiej Zolnierkiewicz #include <linux/ata.h>
85a0e3ad6STejun Heo #include <linux/slab.h>
91da177e4SLinus Torvalds #include <linux/hdreg.h>
103582dd29SJens Axboe #include <linux/blk-mq.h>
111da177e4SLinus Torvalds #include <linux/skbuff.h>
121da177e4SLinus Torvalds #include <linux/netdevice.h>
1368e0d42fSEd L. Cashin #include <linux/moduleparam.h>
14896831f5SEd Cashin #include <linux/workqueue.h>
15896831f5SEd Cashin #include <linux/kthread.h>
16881d966bSEric W. Biederman #include <net/net_namespace.h>
17475172fbSEd L. Cashin #include <asm/unaligned.h>
18896831f5SEd Cashin #include <linux/uio.h>
191da177e4SLinus Torvalds #include "aoe.h"
201da177e4SLinus Torvalds
21896831f5SEd Cashin #define MAXIOC (8192) /* default meant to avoid most soft lockups */
22896831f5SEd Cashin
23896831f5SEd Cashin static void ktcomplete(struct frame *, struct sk_buff *);
24bbb44e30SEd Cashin static int count_targets(struct aoedev *d, int *untainted);
25896831f5SEd Cashin
2669cf2d85SEd Cashin static struct buf *nextbuf(struct aoedev *);
2769cf2d85SEd Cashin
28b751e8b6SEd L. Cashin static int aoe_deadsecs = 60 * 3;
29b751e8b6SEd L. Cashin module_param(aoe_deadsecs, int, 0644);
30b751e8b6SEd L. Cashin MODULE_PARM_DESC(aoe_deadsecs, "After aoe_deadsecs seconds, give up and fail dev.");
311da177e4SLinus Torvalds
327b6ccc5fSEd Cashin static int aoe_maxout = 64;
337df620d8SEd L. Cashin module_param(aoe_maxout, int, 0644);
347df620d8SEd L. Cashin MODULE_PARM_DESC(aoe_maxout,
357df620d8SEd L. Cashin "Only aoe_maxout outstanding packets for every MAC on eX.Y.");
367df620d8SEd L. Cashin
378030d343SEd Cashin /* The number of online cpus during module initialization gives us a
388030d343SEd Cashin * convenient heuristic cap on the parallelism used for ktio threads
398030d343SEd Cashin * doing I/O completion. It is not important that the cap equal the
408030d343SEd Cashin * actual number of running CPUs at any given time, but because of CPU
418030d343SEd Cashin * hotplug, we take care to use ncpus instead of using
428030d343SEd Cashin * num_online_cpus() after module initialization.
438030d343SEd Cashin */
448030d343SEd Cashin static int ncpus;
458030d343SEd Cashin
468030d343SEd Cashin /* mutex lock used for synchronization while thread spawning */
478030d343SEd Cashin static DEFINE_MUTEX(ktio_spawn_lock);
488030d343SEd Cashin
498030d343SEd Cashin static wait_queue_head_t *ktiowq;
508030d343SEd Cashin static struct ktstate *kts;
51896831f5SEd Cashin
52896831f5SEd Cashin /* io completion queue */
538030d343SEd Cashin struct iocq_ktio {
54896831f5SEd Cashin struct list_head head;
55896831f5SEd Cashin spinlock_t lock;
568030d343SEd Cashin };
578030d343SEd Cashin static struct iocq_ktio *iocq;
58896831f5SEd Cashin
59bbb44e30SEd Cashin static struct page *empty_page;
60bbb44e30SEd Cashin
6168e0d42fSEd L. Cashin static struct sk_buff *
new_skb(ulong len)62e407a7f6SEd L. Cashin new_skb(ulong len)
631da177e4SLinus Torvalds {
641da177e4SLinus Torvalds struct sk_buff *skb;
651da177e4SLinus Torvalds
6691c57464SEric Dumazet skb = alloc_skb(len + MAX_HEADER, GFP_ATOMIC);
671da177e4SLinus Torvalds if (skb) {
6891c57464SEric Dumazet skb_reserve(skb, MAX_HEADER);
69459a98edSArnaldo Carvalho de Melo skb_reset_mac_header(skb);
70c1d2bbe1SArnaldo Carvalho de Melo skb_reset_network_header(skb);
711da177e4SLinus Torvalds skb->protocol = __constant_htons(ETH_P_AOE);
728babe8ccSEd Cashin skb_checksum_none_assert(skb);
731da177e4SLinus Torvalds }
741da177e4SLinus Torvalds return skb;
751da177e4SLinus Torvalds }
761da177e4SLinus Torvalds
771da177e4SLinus Torvalds static struct frame *
getframe_deferred(struct aoedev * d,u32 tag)783a0c40d2SEd Cashin getframe_deferred(struct aoedev *d, u32 tag)
793a0c40d2SEd Cashin {
803a0c40d2SEd Cashin struct list_head *head, *pos, *nx;
813a0c40d2SEd Cashin struct frame *f;
823a0c40d2SEd Cashin
833a0c40d2SEd Cashin head = &d->rexmitq;
843a0c40d2SEd Cashin list_for_each_safe(pos, nx, head) {
853a0c40d2SEd Cashin f = list_entry(pos, struct frame, head);
863a0c40d2SEd Cashin if (f->tag == tag) {
873a0c40d2SEd Cashin list_del(pos);
883a0c40d2SEd Cashin return f;
893a0c40d2SEd Cashin }
903a0c40d2SEd Cashin }
913a0c40d2SEd Cashin return NULL;
923a0c40d2SEd Cashin }
933a0c40d2SEd Cashin
943a0c40d2SEd Cashin static struct frame *
getframe(struct aoedev * d,u32 tag)9564a80f5aSEd Cashin getframe(struct aoedev *d, u32 tag)
961da177e4SLinus Torvalds {
97896831f5SEd Cashin struct frame *f;
98896831f5SEd Cashin struct list_head *head, *pos, *nx;
99896831f5SEd Cashin u32 n;
1001da177e4SLinus Torvalds
101896831f5SEd Cashin n = tag % NFACTIVE;
10264a80f5aSEd Cashin head = &d->factive[n];
103896831f5SEd Cashin list_for_each_safe(pos, nx, head) {
104896831f5SEd Cashin f = list_entry(pos, struct frame, head);
105896831f5SEd Cashin if (f->tag == tag) {
106896831f5SEd Cashin list_del(pos);
1071da177e4SLinus Torvalds return f;
108896831f5SEd Cashin }
109896831f5SEd Cashin }
1101da177e4SLinus Torvalds return NULL;
1111da177e4SLinus Torvalds }
1121da177e4SLinus Torvalds
1131da177e4SLinus Torvalds /*
1141da177e4SLinus Torvalds * Leave the top bit clear so we have tagspace for userland.
1151da177e4SLinus Torvalds * The bottom 16 bits are the xmit tick for rexmit/rttavg processing.
1161da177e4SLinus Torvalds * This driver reserves tag -1 to mean "unused frame."
1171da177e4SLinus Torvalds */
1181da177e4SLinus Torvalds static int
newtag(struct aoedev * d)11964a80f5aSEd Cashin newtag(struct aoedev *d)
1201da177e4SLinus Torvalds {
1211da177e4SLinus Torvalds register ulong n;
1221da177e4SLinus Torvalds
1231da177e4SLinus Torvalds n = jiffies & 0xffff;
124a6431e35SColin Ian King return n | (++d->lasttag & 0x7fff) << 16;
1251da177e4SLinus Torvalds }
1261da177e4SLinus Torvalds
127896831f5SEd Cashin static u32
aoehdr_atainit(struct aoedev * d,struct aoetgt * t,struct aoe_hdr * h)12868e0d42fSEd L. Cashin aoehdr_atainit(struct aoedev *d, struct aoetgt *t, struct aoe_hdr *h)
1291da177e4SLinus Torvalds {
13064a80f5aSEd Cashin u32 host_tag = newtag(d);
1311da177e4SLinus Torvalds
13268e0d42fSEd L. Cashin memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src);
13368e0d42fSEd L. Cashin memcpy(h->dst, t->addr, sizeof h->dst);
13463e9cc5dSecashin@coraid.com h->type = __constant_cpu_to_be16(ETH_P_AOE);
1351da177e4SLinus Torvalds h->verfl = AOE_HVER;
13663e9cc5dSecashin@coraid.com h->major = cpu_to_be16(d->aoemajor);
1371da177e4SLinus Torvalds h->minor = d->aoeminor;
1381da177e4SLinus Torvalds h->cmd = AOECMD_ATA;
13963e9cc5dSecashin@coraid.com h->tag = cpu_to_be32(host_tag);
1401da177e4SLinus Torvalds
1411da177e4SLinus Torvalds return host_tag;
1421da177e4SLinus Torvalds }
1431da177e4SLinus Torvalds
14419bf2635SEd L. Cashin static inline void
put_lba(struct aoe_atahdr * ah,sector_t lba)14519bf2635SEd L. Cashin put_lba(struct aoe_atahdr *ah, sector_t lba)
14619bf2635SEd L. Cashin {
14719bf2635SEd L. Cashin ah->lba0 = lba;
14819bf2635SEd L. Cashin ah->lba1 = lba >>= 8;
14919bf2635SEd L. Cashin ah->lba2 = lba >>= 8;
15019bf2635SEd L. Cashin ah->lba3 = lba >>= 8;
15119bf2635SEd L. Cashin ah->lba4 = lba >>= 8;
15219bf2635SEd L. Cashin ah->lba5 = lba >>= 8;
15319bf2635SEd L. Cashin }
15419bf2635SEd L. Cashin
1553f0f0133SEd Cashin static struct aoeif *
ifrotate(struct aoetgt * t)15668e0d42fSEd L. Cashin ifrotate(struct aoetgt *t)
1571da177e4SLinus Torvalds {
1583f0f0133SEd Cashin struct aoeif *ifp;
1593f0f0133SEd Cashin
1603f0f0133SEd Cashin ifp = t->ifp;
1613f0f0133SEd Cashin ifp++;
1623f0f0133SEd Cashin if (ifp >= &t->ifs[NAOEIFS] || ifp->nd == NULL)
1633f0f0133SEd Cashin ifp = t->ifs;
1643f0f0133SEd Cashin if (ifp->nd == NULL)
1653f0f0133SEd Cashin return NULL;
1663f0f0133SEd Cashin return t->ifp = ifp;
16768e0d42fSEd L. Cashin }
16868e0d42fSEd L. Cashin
1699bb237b6SEd L. Cashin static void
skb_pool_put(struct aoedev * d,struct sk_buff * skb)1709bb237b6SEd L. Cashin skb_pool_put(struct aoedev *d, struct sk_buff *skb)
1719bb237b6SEd L. Cashin {
172e9bb8fb0SDavid S. Miller __skb_queue_tail(&d->skbpool, skb);
1739bb237b6SEd L. Cashin }
1749bb237b6SEd L. Cashin
1759bb237b6SEd L. Cashin static struct sk_buff *
skb_pool_get(struct aoedev * d)1769bb237b6SEd L. Cashin skb_pool_get(struct aoedev *d)
1779bb237b6SEd L. Cashin {
178e9bb8fb0SDavid S. Miller struct sk_buff *skb = skb_peek(&d->skbpool);
1799bb237b6SEd L. Cashin
1809bb237b6SEd L. Cashin if (skb && atomic_read(&skb_shinfo(skb)->dataref) == 1) {
181e9bb8fb0SDavid S. Miller __skb_unlink(skb, &d->skbpool);
1829bb237b6SEd L. Cashin return skb;
1839bb237b6SEd L. Cashin }
184e9bb8fb0SDavid S. Miller if (skb_queue_len(&d->skbpool) < NSKBPOOLMAX &&
185e9bb8fb0SDavid S. Miller (skb = new_skb(ETH_ZLEN)))
1869bb237b6SEd L. Cashin return skb;
187e9bb8fb0SDavid S. Miller
1889bb237b6SEd L. Cashin return NULL;
1899bb237b6SEd L. Cashin }
1909bb237b6SEd L. Cashin
191896831f5SEd Cashin void
aoe_freetframe(struct frame * f)192896831f5SEd Cashin aoe_freetframe(struct frame *f)
19368e0d42fSEd L. Cashin {
194896831f5SEd Cashin struct aoetgt *t;
195896831f5SEd Cashin
196896831f5SEd Cashin t = f->t;
197896831f5SEd Cashin f->buf = NULL;
198feb261e2SKent Overstreet memset(&f->iter, 0, sizeof(f->iter));
199896831f5SEd Cashin f->r_skb = NULL;
200bbb44e30SEd Cashin f->flags = 0;
201896831f5SEd Cashin list_add(&f->head, &t->ffree);
202896831f5SEd Cashin }
203896831f5SEd Cashin
204896831f5SEd Cashin static struct frame *
newtframe(struct aoedev * d,struct aoetgt * t)205896831f5SEd Cashin newtframe(struct aoedev *d, struct aoetgt *t)
206896831f5SEd Cashin {
207896831f5SEd Cashin struct frame *f;
2089bb237b6SEd L. Cashin struct sk_buff *skb;
209896831f5SEd Cashin struct list_head *pos;
210896831f5SEd Cashin
211896831f5SEd Cashin if (list_empty(&t->ffree)) {
212896831f5SEd Cashin if (t->falloc >= NSKBPOOLMAX*2)
213896831f5SEd Cashin return NULL;
214896831f5SEd Cashin f = kcalloc(1, sizeof(*f), GFP_ATOMIC);
215896831f5SEd Cashin if (f == NULL)
216896831f5SEd Cashin return NULL;
217896831f5SEd Cashin t->falloc++;
218896831f5SEd Cashin f->t = t;
219896831f5SEd Cashin } else {
220896831f5SEd Cashin pos = t->ffree.next;
221896831f5SEd Cashin list_del(pos);
222896831f5SEd Cashin f = list_entry(pos, struct frame, head);
223896831f5SEd Cashin }
224896831f5SEd Cashin
225896831f5SEd Cashin skb = f->skb;
226896831f5SEd Cashin if (skb == NULL) {
227896831f5SEd Cashin f->skb = skb = new_skb(ETH_ZLEN);
228896831f5SEd Cashin if (!skb) {
229896831f5SEd Cashin bail: aoe_freetframe(f);
230896831f5SEd Cashin return NULL;
231896831f5SEd Cashin }
232896831f5SEd Cashin }
233896831f5SEd Cashin
234896831f5SEd Cashin if (atomic_read(&skb_shinfo(skb)->dataref) != 1) {
235896831f5SEd Cashin skb = skb_pool_get(d);
236896831f5SEd Cashin if (skb == NULL)
237896831f5SEd Cashin goto bail;
238896831f5SEd Cashin skb_pool_put(d, f->skb);
239896831f5SEd Cashin f->skb = skb;
240896831f5SEd Cashin }
241896831f5SEd Cashin
242896831f5SEd Cashin skb->truesize -= skb->data_len;
243896831f5SEd Cashin skb_shinfo(skb)->nr_frags = skb->data_len = 0;
244896831f5SEd Cashin skb_trim(skb, 0);
245896831f5SEd Cashin return f;
246896831f5SEd Cashin }
247896831f5SEd Cashin
248896831f5SEd Cashin static struct frame *
newframe(struct aoedev * d)249896831f5SEd Cashin newframe(struct aoedev *d)
250896831f5SEd Cashin {
251896831f5SEd Cashin struct frame *f;
252896831f5SEd Cashin struct aoetgt *t, **tt;
253896831f5SEd Cashin int totout = 0;
254bbb44e30SEd Cashin int use_tainted;
255bbb44e30SEd Cashin int has_untainted;
25668e0d42fSEd L. Cashin
25771114ec4SEd Cashin if (!d->targets || !d->targets[0]) {
25868e0d42fSEd L. Cashin printk(KERN_ERR "aoe: NULL TARGETS!\n");
25968e0d42fSEd L. Cashin return NULL;
26068e0d42fSEd L. Cashin }
261896831f5SEd Cashin tt = d->tgt; /* last used target */
262bbb44e30SEd Cashin for (use_tainted = 0, has_untainted = 0;;) {
263896831f5SEd Cashin tt++;
26471114ec4SEd Cashin if (tt >= &d->targets[d->ntargets] || !*tt)
265896831f5SEd Cashin tt = d->targets;
266896831f5SEd Cashin t = *tt;
267bbb44e30SEd Cashin if (!t->taint) {
268bbb44e30SEd Cashin has_untainted = 1;
269896831f5SEd Cashin totout += t->nout;
270bbb44e30SEd Cashin }
271896831f5SEd Cashin if (t->nout < t->maxout
272bbb44e30SEd Cashin && (use_tainted || !t->taint)
273896831f5SEd Cashin && t->ifp->nd) {
274896831f5SEd Cashin f = newtframe(d, t);
275896831f5SEd Cashin if (f) {
276896831f5SEd Cashin ifrotate(t);
2773f0f0133SEd Cashin d->tgt = tt;
27868e0d42fSEd L. Cashin return f;
27968e0d42fSEd L. Cashin }
2809bb237b6SEd L. Cashin }
281bbb44e30SEd Cashin if (tt == d->tgt) { /* we've looped and found nada */
282bbb44e30SEd Cashin if (!use_tainted && !has_untainted)
283bbb44e30SEd Cashin use_tainted = 1;
284bbb44e30SEd Cashin else
2859bb237b6SEd L. Cashin break;
286896831f5SEd Cashin }
287bbb44e30SEd Cashin }
288896831f5SEd Cashin if (totout == 0) {
289896831f5SEd Cashin d->kicked++;
290896831f5SEd Cashin d->flags |= DEVFL_KICKME;
2919bb237b6SEd L. Cashin }
29268e0d42fSEd L. Cashin return NULL;
29368e0d42fSEd L. Cashin }
29468e0d42fSEd L. Cashin
2953d5b0605SEd Cashin static void
skb_fillup(struct sk_buff * skb,struct bio * bio,struct bvec_iter iter)296feb261e2SKent Overstreet skb_fillup(struct sk_buff *skb, struct bio *bio, struct bvec_iter iter)
2973d5b0605SEd Cashin {
2983d5b0605SEd Cashin int frag = 0;
299feb261e2SKent Overstreet struct bio_vec bv;
300feb261e2SKent Overstreet
301feb261e2SKent Overstreet __bio_for_each_segment(bv, bio, iter, iter)
302feb261e2SKent Overstreet skb_fill_page_desc(skb, frag++, bv.bv_page,
303feb261e2SKent Overstreet bv.bv_offset, bv.bv_len);
3043d5b0605SEd Cashin }
3053d5b0605SEd Cashin
306896831f5SEd Cashin static void
fhash(struct frame * f)307896831f5SEd Cashin fhash(struct frame *f)
308896831f5SEd Cashin {
30964a80f5aSEd Cashin struct aoedev *d = f->t->d;
310896831f5SEd Cashin u32 n;
311896831f5SEd Cashin
312896831f5SEd Cashin n = f->tag % NFACTIVE;
31364a80f5aSEd Cashin list_add_tail(&f->head, &d->factive[n]);
314896831f5SEd Cashin }
315896831f5SEd Cashin
316bbb44e30SEd Cashin static void
ata_rw_frameinit(struct frame * f)317bbb44e30SEd Cashin ata_rw_frameinit(struct frame *f)
318bbb44e30SEd Cashin {
319bbb44e30SEd Cashin struct aoetgt *t;
320bbb44e30SEd Cashin struct aoe_hdr *h;
321bbb44e30SEd Cashin struct aoe_atahdr *ah;
322bbb44e30SEd Cashin struct sk_buff *skb;
323bbb44e30SEd Cashin char writebit, extbit;
324bbb44e30SEd Cashin
325bbb44e30SEd Cashin skb = f->skb;
326bbb44e30SEd Cashin h = (struct aoe_hdr *) skb_mac_header(skb);
327bbb44e30SEd Cashin ah = (struct aoe_atahdr *) (h + 1);
328bbb44e30SEd Cashin skb_put(skb, sizeof(*h) + sizeof(*ah));
329bbb44e30SEd Cashin memset(h, 0, skb->len);
330bbb44e30SEd Cashin
331bbb44e30SEd Cashin writebit = 0x10;
332bbb44e30SEd Cashin extbit = 0x4;
333bbb44e30SEd Cashin
334bbb44e30SEd Cashin t = f->t;
335bbb44e30SEd Cashin f->tag = aoehdr_atainit(t->d, t, h);
336bbb44e30SEd Cashin fhash(f);
337bbb44e30SEd Cashin t->nout++;
338bbb44e30SEd Cashin f->waited = 0;
339bbb44e30SEd Cashin f->waited_total = 0;
340bbb44e30SEd Cashin
341bbb44e30SEd Cashin /* set up ata header */
342feb261e2SKent Overstreet ah->scnt = f->iter.bi_size >> 9;
343feb261e2SKent Overstreet put_lba(ah, f->iter.bi_sector);
344bbb44e30SEd Cashin if (t->d->flags & DEVFL_EXT) {
345bbb44e30SEd Cashin ah->aflags |= AOEAFL_EXT;
346bbb44e30SEd Cashin } else {
347bbb44e30SEd Cashin extbit = 0;
348bbb44e30SEd Cashin ah->lba3 &= 0x0f;
349bbb44e30SEd Cashin ah->lba3 |= 0xe0; /* LBA bit + obsolete 0xa0 */
350bbb44e30SEd Cashin }
351bbb44e30SEd Cashin if (f->buf && bio_data_dir(f->buf->bio) == WRITE) {
352feb261e2SKent Overstreet skb_fillup(skb, f->buf->bio, f->iter);
353bbb44e30SEd Cashin ah->aflags |= AOEAFL_WRITE;
354feb261e2SKent Overstreet skb->len += f->iter.bi_size;
355feb261e2SKent Overstreet skb->data_len = f->iter.bi_size;
356feb261e2SKent Overstreet skb->truesize += f->iter.bi_size;
357bbb44e30SEd Cashin t->wpkts++;
358bbb44e30SEd Cashin } else {
359bbb44e30SEd Cashin t->rpkts++;
360bbb44e30SEd Cashin writebit = 0;
361bbb44e30SEd Cashin }
362bbb44e30SEd Cashin
363bbb44e30SEd Cashin ah->cmdstat = ATA_CMD_PIO_READ | writebit | extbit;
364bbb44e30SEd Cashin skb->dev = t->ifp->nd;
365bbb44e30SEd Cashin }
366bbb44e30SEd Cashin
36768e0d42fSEd L. Cashin static int
aoecmd_ata_rw(struct aoedev * d)36868e0d42fSEd L. Cashin aoecmd_ata_rw(struct aoedev *d)
36968e0d42fSEd L. Cashin {
37068e0d42fSEd L. Cashin struct frame *f;
3711da177e4SLinus Torvalds struct buf *buf;
3721da177e4SLinus Torvalds struct sk_buff *skb;
37369cf2d85SEd Cashin struct sk_buff_head queue;
3741da177e4SLinus Torvalds
37569cf2d85SEd Cashin buf = nextbuf(d);
37669cf2d85SEd Cashin if (buf == NULL)
37769cf2d85SEd Cashin return 0;
378896831f5SEd Cashin f = newframe(d);
37968e0d42fSEd L. Cashin if (f == NULL)
38068e0d42fSEd L. Cashin return 0;
3813d5b0605SEd Cashin
3821da177e4SLinus Torvalds /* initialize the headers & frame */
3831da177e4SLinus Torvalds f->buf = buf;
384feb261e2SKent Overstreet f->iter = buf->iter;
385feb261e2SKent Overstreet f->iter.bi_size = min_t(unsigned long,
386feb261e2SKent Overstreet d->maxbcnt ?: DEFAULTBCNT,
387feb261e2SKent Overstreet f->iter.bi_size);
388feb261e2SKent Overstreet bio_advance_iter(buf->bio, &buf->iter, f->iter.bi_size);
389feb261e2SKent Overstreet
390feb261e2SKent Overstreet if (!buf->iter.bi_size)
391feb261e2SKent Overstreet d->ip.buf = NULL;
3921da177e4SLinus Torvalds
3931da177e4SLinus Torvalds /* mark all tracking fields and load out */
3941da177e4SLinus Torvalds buf->nframesout += 1;
395feb261e2SKent Overstreet
396feb261e2SKent Overstreet ata_rw_frameinit(f);
3971da177e4SLinus Torvalds
398bbb44e30SEd Cashin skb = skb_clone(f->skb, GFP_ATOMIC);
39969cf2d85SEd Cashin if (skb) {
40085cf955dSTina Ruchandani f->sent = ktime_get();
40169cf2d85SEd Cashin __skb_queue_head_init(&queue);
40269cf2d85SEd Cashin __skb_queue_tail(&queue, skb);
40369cf2d85SEd Cashin aoenet_xmit(&queue);
40469cf2d85SEd Cashin }
40568e0d42fSEd L. Cashin return 1;
40668e0d42fSEd L. Cashin }
4071da177e4SLinus Torvalds
4083ae1c24eSEd L. Cashin /* some callers cannot sleep, and they can call this function,
4093ae1c24eSEd L. Cashin * transmitting the packets later, when interrupts are on
4103ae1c24eSEd L. Cashin */
411e9bb8fb0SDavid S. Miller static void
aoecmd_cfg_pkts(ushort aoemajor,unsigned char aoeminor,struct sk_buff_head * queue)412e9bb8fb0SDavid S. Miller aoecmd_cfg_pkts(ushort aoemajor, unsigned char aoeminor, struct sk_buff_head *queue)
4133ae1c24eSEd L. Cashin {
4143ae1c24eSEd L. Cashin struct aoe_hdr *h;
4153ae1c24eSEd L. Cashin struct aoe_cfghdr *ch;
416e9bb8fb0SDavid S. Miller struct sk_buff *skb;
4173ae1c24eSEd L. Cashin struct net_device *ifp;
4183ae1c24eSEd L. Cashin
419840a185dSEric Dumazet rcu_read_lock();
420840a185dSEric Dumazet for_each_netdev_rcu(&init_net, ifp) {
4213ae1c24eSEd L. Cashin dev_hold(ifp);
422*eb48680bSChun-Yi Lee if (!is_aoe_netif(ifp)) {
423*eb48680bSChun-Yi Lee dev_put(ifp);
424*eb48680bSChun-Yi Lee continue;
425*eb48680bSChun-Yi Lee }
4263ae1c24eSEd L. Cashin
427e407a7f6SEd L. Cashin skb = new_skb(sizeof *h + sizeof *ch);
4283ae1c24eSEd L. Cashin if (skb == NULL) {
429a12c93f0SEd L. Cashin printk(KERN_INFO "aoe: skb alloc failure\n");
430*eb48680bSChun-Yi Lee dev_put(ifp);
431*eb48680bSChun-Yi Lee continue;
4323ae1c24eSEd L. Cashin }
43319900cdeSEd L. Cashin skb_put(skb, sizeof *h + sizeof *ch);
434e407a7f6SEd L. Cashin skb->dev = ifp;
435e9bb8fb0SDavid S. Miller __skb_queue_tail(queue, skb);
436abdbf94dSEd L. Cashin h = (struct aoe_hdr *) skb_mac_header(skb);
4373ae1c24eSEd L. Cashin memset(h, 0, sizeof *h + sizeof *ch);
4383ae1c24eSEd L. Cashin
4393ae1c24eSEd L. Cashin memset(h->dst, 0xff, sizeof h->dst);
4403ae1c24eSEd L. Cashin memcpy(h->src, ifp->dev_addr, sizeof h->src);
4413ae1c24eSEd L. Cashin h->type = __constant_cpu_to_be16(ETH_P_AOE);
4423ae1c24eSEd L. Cashin h->verfl = AOE_HVER;
4433ae1c24eSEd L. Cashin h->major = cpu_to_be16(aoemajor);
4443ae1c24eSEd L. Cashin h->minor = aoeminor;
4453ae1c24eSEd L. Cashin h->cmd = AOECMD_CFG;
4463ae1c24eSEd L. Cashin }
447840a185dSEric Dumazet rcu_read_unlock();
4483ae1c24eSEd L. Cashin }
4493ae1c24eSEd L. Cashin
4501da177e4SLinus Torvalds static void
resend(struct aoedev * d,struct frame * f)451896831f5SEd Cashin resend(struct aoedev *d, struct frame *f)
4521da177e4SLinus Torvalds {
4531da177e4SLinus Torvalds struct sk_buff *skb;
45469cf2d85SEd Cashin struct sk_buff_head queue;
4551da177e4SLinus Torvalds struct aoe_hdr *h;
456896831f5SEd Cashin struct aoetgt *t;
4571da177e4SLinus Torvalds char buf[128];
4581da177e4SLinus Torvalds u32 n;
4591da177e4SLinus Torvalds
460896831f5SEd Cashin t = f->t;
46164a80f5aSEd Cashin n = newtag(d);
462e407a7f6SEd L. Cashin skb = f->skb;
4633f0f0133SEd Cashin if (ifrotate(t) == NULL) {
4643f0f0133SEd Cashin /* probably can't happen, but set it up to fail anyway */
4653f0f0133SEd Cashin pr_info("aoe: resend: no interfaces to rotate to.\n");
4663f0f0133SEd Cashin ktcomplete(f, NULL);
4673f0f0133SEd Cashin return;
4683f0f0133SEd Cashin }
469abdbf94dSEd L. Cashin h = (struct aoe_hdr *) skb_mac_header(skb);
47068e0d42fSEd L. Cashin
471bbb44e30SEd Cashin if (!(f->flags & FFL_PROBE)) {
472bbb44e30SEd Cashin snprintf(buf, sizeof(buf),
473411c41eeSHarvey Harrison "%15s e%ld.%d oldtag=%08x@%08lx newtag=%08x s=%pm d=%pm nout=%d\n",
474bbb44e30SEd Cashin "retransmit", d->aoemajor, d->aoeminor,
475bbb44e30SEd Cashin f->tag, jiffies, n,
476411c41eeSHarvey Harrison h->src, h->dst, t->nout);
47768e0d42fSEd L. Cashin aoechr_error(buf);
478bbb44e30SEd Cashin }
47968e0d42fSEd L. Cashin
4801da177e4SLinus Torvalds f->tag = n;
481896831f5SEd Cashin fhash(f);
48263e9cc5dSecashin@coraid.com h->tag = cpu_to_be32(n);
48368e0d42fSEd L. Cashin memcpy(h->dst, t->addr, sizeof h->dst);
48468e0d42fSEd L. Cashin memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src);
4851da177e4SLinus Torvalds
48668e0d42fSEd L. Cashin skb->dev = t->ifp->nd;
4874f51dc5eSEd L. Cashin skb = skb_clone(skb, GFP_ATOMIC);
4884f51dc5eSEd L. Cashin if (skb == NULL)
4894f51dc5eSEd L. Cashin return;
49085cf955dSTina Ruchandani f->sent = ktime_get();
49169cf2d85SEd Cashin __skb_queue_head_init(&queue);
49269cf2d85SEd Cashin __skb_queue_tail(&queue, skb);
49369cf2d85SEd Cashin aoenet_xmit(&queue);
4941da177e4SLinus Torvalds }
4951da177e4SLinus Torvalds
4961da177e4SLinus Torvalds static int
tsince_hr(struct frame * f)4975f0c9c48SEd Cashin tsince_hr(struct frame *f)
4985f0c9c48SEd Cashin {
49985cf955dSTina Ruchandani u64 delta = ktime_to_ns(ktime_sub(ktime_get(), f->sent));
5005f0c9c48SEd Cashin
50185cf955dSTina Ruchandani /* delta is normally under 4.2 seconds, avoid 64-bit division */
50285cf955dSTina Ruchandani if (likely(delta <= UINT_MAX))
50385cf955dSTina Ruchandani return (u32)delta / NSEC_PER_USEC;
5045f0c9c48SEd Cashin
50585cf955dSTina Ruchandani /* avoid overflow after 71 minutes */
50685cf955dSTina Ruchandani if (delta > ((u64)INT_MAX * NSEC_PER_USEC))
50785cf955dSTina Ruchandani return INT_MAX;
5085f0c9c48SEd Cashin
50985cf955dSTina Ruchandani return div_u64(delta, NSEC_PER_USEC);
5105f0c9c48SEd Cashin }
5115f0c9c48SEd Cashin
5125f0c9c48SEd Cashin static int
tsince(u32 tag)513896831f5SEd Cashin tsince(u32 tag)
5141da177e4SLinus Torvalds {
5151da177e4SLinus Torvalds int n;
5161da177e4SLinus Torvalds
5171da177e4SLinus Torvalds n = jiffies & 0xffff;
5181da177e4SLinus Torvalds n -= tag & 0xffff;
5191da177e4SLinus Torvalds if (n < 0)
5201da177e4SLinus Torvalds n += 1<<16;
5215f0c9c48SEd Cashin return jiffies_to_usecs(n + 1);
5221da177e4SLinus Torvalds }
5231da177e4SLinus Torvalds
52468e0d42fSEd L. Cashin static struct aoeif *
getif(struct aoetgt * t,struct net_device * nd)52568e0d42fSEd L. Cashin getif(struct aoetgt *t, struct net_device *nd)
52668e0d42fSEd L. Cashin {
52768e0d42fSEd L. Cashin struct aoeif *p, *e;
52868e0d42fSEd L. Cashin
52968e0d42fSEd L. Cashin p = t->ifs;
53068e0d42fSEd L. Cashin e = p + NAOEIFS;
53168e0d42fSEd L. Cashin for (; p < e; p++)
53268e0d42fSEd L. Cashin if (p->nd == nd)
53368e0d42fSEd L. Cashin return p;
53468e0d42fSEd L. Cashin return NULL;
53568e0d42fSEd L. Cashin }
53668e0d42fSEd L. Cashin
53768e0d42fSEd L. Cashin static void
ejectif(struct aoetgt * t,struct aoeif * ifp)53868e0d42fSEd L. Cashin ejectif(struct aoetgt *t, struct aoeif *ifp)
53968e0d42fSEd L. Cashin {
54068e0d42fSEd L. Cashin struct aoeif *e;
5411b86fda9SEd Cashin struct net_device *nd;
54268e0d42fSEd L. Cashin ulong n;
54368e0d42fSEd L. Cashin
5441b86fda9SEd Cashin nd = ifp->nd;
54568e0d42fSEd L. Cashin e = t->ifs + NAOEIFS - 1;
54668e0d42fSEd L. Cashin n = (e - ifp) * sizeof *ifp;
54768e0d42fSEd L. Cashin memmove(ifp, ifp+1, n);
54868e0d42fSEd L. Cashin e->nd = NULL;
5491b86fda9SEd Cashin dev_put(nd);
55068e0d42fSEd L. Cashin }
55168e0d42fSEd L. Cashin
5523fc9b032SEd Cashin static struct frame *
reassign_frame(struct frame * f)553bbb44e30SEd Cashin reassign_frame(struct frame *f)
55468e0d42fSEd L. Cashin {
5553fc9b032SEd Cashin struct frame *nf;
55668e0d42fSEd L. Cashin struct sk_buff *skb;
55768e0d42fSEd L. Cashin
5583fc9b032SEd Cashin nf = newframe(f->t->d);
55968e0d42fSEd L. Cashin if (!nf)
5603fc9b032SEd Cashin return NULL;
561bbb44e30SEd Cashin if (nf->t == f->t) {
562bbb44e30SEd Cashin aoe_freetframe(nf);
563bbb44e30SEd Cashin return NULL;
564bbb44e30SEd Cashin }
565896831f5SEd Cashin
56668e0d42fSEd L. Cashin skb = nf->skb;
567896831f5SEd Cashin nf->skb = f->skb;
568896831f5SEd Cashin nf->buf = f->buf;
569feb261e2SKent Overstreet nf->iter = f->iter;
57068e0d42fSEd L. Cashin nf->waited = 0;
5713fc9b032SEd Cashin nf->waited_total = f->waited_total;
5723fc9b032SEd Cashin nf->sent = f->sent;
573896831f5SEd Cashin f->skb = skb;
5743fc9b032SEd Cashin
5753fc9b032SEd Cashin return nf;
5763fc9b032SEd Cashin }
5773fc9b032SEd Cashin
578bbb44e30SEd Cashin static void
probe(struct aoetgt * t)579bbb44e30SEd Cashin probe(struct aoetgt *t)
5803fc9b032SEd Cashin {
581bbb44e30SEd Cashin struct aoedev *d;
582bbb44e30SEd Cashin struct frame *f;
583bbb44e30SEd Cashin struct sk_buff *skb;
584bbb44e30SEd Cashin struct sk_buff_head queue;
585bbb44e30SEd Cashin size_t n, m;
586bbb44e30SEd Cashin int frag;
5873fc9b032SEd Cashin
588bbb44e30SEd Cashin d = t->d;
589bbb44e30SEd Cashin f = newtframe(d, t);
590bbb44e30SEd Cashin if (!f) {
591bbb44e30SEd Cashin pr_err("%s %pm for e%ld.%d: %s\n",
592bbb44e30SEd Cashin "aoe: cannot probe remote address",
593bbb44e30SEd Cashin t->addr,
594bbb44e30SEd Cashin (long) d->aoemajor, d->aoeminor,
595bbb44e30SEd Cashin "no frame available");
596bbb44e30SEd Cashin return;
597bbb44e30SEd Cashin }
598bbb44e30SEd Cashin f->flags |= FFL_PROBE;
599bbb44e30SEd Cashin ifrotate(t);
600feb261e2SKent Overstreet f->iter.bi_size = t->d->maxbcnt ? t->d->maxbcnt : DEFAULTBCNT;
601bbb44e30SEd Cashin ata_rw_frameinit(f);
602bbb44e30SEd Cashin skb = f->skb;
603feb261e2SKent Overstreet for (frag = 0, n = f->iter.bi_size; n > 0; ++frag, n -= m) {
604bbb44e30SEd Cashin if (n < PAGE_SIZE)
605bbb44e30SEd Cashin m = n;
606bbb44e30SEd Cashin else
607bbb44e30SEd Cashin m = PAGE_SIZE;
608bbb44e30SEd Cashin skb_fill_page_desc(skb, frag, empty_page, 0, m);
609bbb44e30SEd Cashin }
610feb261e2SKent Overstreet skb->len += f->iter.bi_size;
611feb261e2SKent Overstreet skb->data_len = f->iter.bi_size;
612feb261e2SKent Overstreet skb->truesize += f->iter.bi_size;
613bbb44e30SEd Cashin
614bbb44e30SEd Cashin skb = skb_clone(f->skb, GFP_ATOMIC);
615bbb44e30SEd Cashin if (skb) {
61685cf955dSTina Ruchandani f->sent = ktime_get();
617bbb44e30SEd Cashin __skb_queue_head_init(&queue);
618bbb44e30SEd Cashin __skb_queue_tail(&queue, skb);
619bbb44e30SEd Cashin aoenet_xmit(&queue);
620896831f5SEd Cashin }
62168e0d42fSEd L. Cashin }
622bbb44e30SEd Cashin
623bbb44e30SEd Cashin static long
rto(struct aoedev * d)624bbb44e30SEd Cashin rto(struct aoedev *d)
625bbb44e30SEd Cashin {
626bbb44e30SEd Cashin long t;
627bbb44e30SEd Cashin
628bbb44e30SEd Cashin t = 2 * d->rttavg >> RTTSCALE;
629bbb44e30SEd Cashin t += 8 * d->rttdev >> RTTDSCALE;
630bbb44e30SEd Cashin if (t == 0)
631bbb44e30SEd Cashin t = 1;
632bbb44e30SEd Cashin
633bbb44e30SEd Cashin return t;
63468e0d42fSEd L. Cashin }
63568e0d42fSEd L. Cashin
6361da177e4SLinus Torvalds static void
rexmit_deferred(struct aoedev * d)6373a0c40d2SEd Cashin rexmit_deferred(struct aoedev *d)
6383a0c40d2SEd Cashin {
6393a0c40d2SEd Cashin struct aoetgt *t;
6403a0c40d2SEd Cashin struct frame *f;
641bbb44e30SEd Cashin struct frame *nf;
6423a0c40d2SEd Cashin struct list_head *pos, *nx, *head;
6433fc9b032SEd Cashin int since;
644bbb44e30SEd Cashin int untainted;
645bbb44e30SEd Cashin
646bbb44e30SEd Cashin count_targets(d, &untainted);
6473a0c40d2SEd Cashin
6483a0c40d2SEd Cashin head = &d->rexmitq;
6493a0c40d2SEd Cashin list_for_each_safe(pos, nx, head) {
6503a0c40d2SEd Cashin f = list_entry(pos, struct frame, head);
6513a0c40d2SEd Cashin t = f->t;
652bbb44e30SEd Cashin if (t->taint) {
653bbb44e30SEd Cashin if (!(f->flags & FFL_PROBE)) {
654bbb44e30SEd Cashin nf = reassign_frame(f);
655bbb44e30SEd Cashin if (nf) {
656bbb44e30SEd Cashin if (t->nout_probes == 0
657bbb44e30SEd Cashin && untainted > 0) {
658bbb44e30SEd Cashin probe(t);
659bbb44e30SEd Cashin t->nout_probes++;
660bbb44e30SEd Cashin }
661bbb44e30SEd Cashin list_replace(&f->head, &nf->head);
662bbb44e30SEd Cashin pos = &nf->head;
663bbb44e30SEd Cashin aoe_freetframe(f);
664bbb44e30SEd Cashin f = nf;
665bbb44e30SEd Cashin t = f->t;
666bbb44e30SEd Cashin }
667bbb44e30SEd Cashin } else if (untainted < 1) {
668bbb44e30SEd Cashin /* don't probe w/o other untainted aoetgts */
669bbb44e30SEd Cashin goto stop_probe;
670bbb44e30SEd Cashin } else if (tsince_hr(f) < t->taint * rto(d)) {
671bbb44e30SEd Cashin /* reprobe slowly when taint is high */
672bbb44e30SEd Cashin continue;
673bbb44e30SEd Cashin }
674bbb44e30SEd Cashin } else if (f->flags & FFL_PROBE) {
675bbb44e30SEd Cashin stop_probe: /* don't probe untainted aoetgts */
676bbb44e30SEd Cashin list_del(pos);
677bbb44e30SEd Cashin aoe_freetframe(f);
678bbb44e30SEd Cashin /* leaving d->kicked, because this is routine */
679bbb44e30SEd Cashin f->t->d->flags |= DEVFL_KICKME;
680bbb44e30SEd Cashin continue;
681bbb44e30SEd Cashin }
6823a0c40d2SEd Cashin if (t->nout >= t->maxout)
6833a0c40d2SEd Cashin continue;
6843a0c40d2SEd Cashin list_del(pos);
6853a0c40d2SEd Cashin t->nout++;
686bbb44e30SEd Cashin if (f->flags & FFL_PROBE)
687bbb44e30SEd Cashin t->nout_probes++;
6883fc9b032SEd Cashin since = tsince_hr(f);
6893fc9b032SEd Cashin f->waited += since;
6903fc9b032SEd Cashin f->waited_total += since;
6913a0c40d2SEd Cashin resend(d, f);
6923a0c40d2SEd Cashin }
6933a0c40d2SEd Cashin }
6943a0c40d2SEd Cashin
695bbb44e30SEd Cashin /* An aoetgt accumulates demerits quickly, and successful
696bbb44e30SEd Cashin * probing redeems the aoetgt slowly.
697bbb44e30SEd Cashin */
698bbb44e30SEd Cashin static void
scorn(struct aoetgt * t)699bbb44e30SEd Cashin scorn(struct aoetgt *t)
700bbb44e30SEd Cashin {
701bbb44e30SEd Cashin int n;
702bbb44e30SEd Cashin
703bbb44e30SEd Cashin n = t->taint++;
704bbb44e30SEd Cashin t->taint += t->taint * 2;
705bbb44e30SEd Cashin if (n > t->taint)
706bbb44e30SEd Cashin t->taint = n;
707bbb44e30SEd Cashin if (t->taint > MAX_TAINT)
708bbb44e30SEd Cashin t->taint = MAX_TAINT;
709bbb44e30SEd Cashin }
710bbb44e30SEd Cashin
711bbb44e30SEd Cashin static int
count_targets(struct aoedev * d,int * untainted)712bbb44e30SEd Cashin count_targets(struct aoedev *d, int *untainted)
713bbb44e30SEd Cashin {
714bbb44e30SEd Cashin int i, good;
715bbb44e30SEd Cashin
716bbb44e30SEd Cashin for (i = good = 0; i < d->ntargets && d->targets[i]; ++i)
717bbb44e30SEd Cashin if (d->targets[i]->taint == 0)
718bbb44e30SEd Cashin good++;
719bbb44e30SEd Cashin
720bbb44e30SEd Cashin if (untainted)
721bbb44e30SEd Cashin *untainted = good;
722bbb44e30SEd Cashin return i;
723bbb44e30SEd Cashin }
724bbb44e30SEd Cashin
7253a0c40d2SEd Cashin static void
rexmit_timer(struct timer_list * timer)7260e0cc9dfSKees Cook rexmit_timer(struct timer_list *timer)
7271da177e4SLinus Torvalds {
7281da177e4SLinus Torvalds struct aoedev *d;
7293a0c40d2SEd Cashin struct aoetgt *t;
73068e0d42fSEd L. Cashin struct aoeif *ifp;
731896831f5SEd Cashin struct frame *f;
732896831f5SEd Cashin struct list_head *head, *pos, *nx;
733896831f5SEd Cashin LIST_HEAD(flist);
7341da177e4SLinus Torvalds register long timeout;
7351da177e4SLinus Torvalds ulong flags, n;
736896831f5SEd Cashin int i;
737bbb44e30SEd Cashin int utgts; /* number of aoetgt descriptors (not slots) */
7383fc9b032SEd Cashin int since;
7391da177e4SLinus Torvalds
7400e0cc9dfSKees Cook d = from_timer(d, timer, timer);
7411da177e4SLinus Torvalds
7420d555ecfSEd Cashin spin_lock_irqsave(&d->lock, flags);
7430d555ecfSEd Cashin
7443a0c40d2SEd Cashin /* timeout based on observed timings and variations */
745bbb44e30SEd Cashin timeout = rto(d);
746bbb44e30SEd Cashin
747bbb44e30SEd Cashin utgts = count_targets(d, NULL);
7481da177e4SLinus Torvalds
7491da177e4SLinus Torvalds if (d->flags & DEVFL_TKILL) {
7501c6f3fcaSEd L. Cashin spin_unlock_irqrestore(&d->lock, flags);
7511da177e4SLinus Torvalds return;
7521da177e4SLinus Torvalds }
753896831f5SEd Cashin
754896831f5SEd Cashin /* collect all frames to rexmit into flist */
755896831f5SEd Cashin for (i = 0; i < NFACTIVE; i++) {
75664a80f5aSEd Cashin head = &d->factive[i];
757896831f5SEd Cashin list_for_each_safe(pos, nx, head) {
758896831f5SEd Cashin f = list_entry(pos, struct frame, head);
7595f0c9c48SEd Cashin if (tsince_hr(f) < timeout)
76064a80f5aSEd Cashin break; /* end of expired frames */
761896831f5SEd Cashin /* move to flist for later processing */
762896831f5SEd Cashin list_move_tail(pos, &flist);
763896831f5SEd Cashin }
764896831f5SEd Cashin }
76569cf2d85SEd Cashin
766896831f5SEd Cashin /* process expired frames */
767896831f5SEd Cashin while (!list_empty(&flist)) {
768896831f5SEd Cashin pos = flist.next;
769896831f5SEd Cashin f = list_entry(pos, struct frame, head);
7703fc9b032SEd Cashin since = tsince_hr(f);
7713fc9b032SEd Cashin n = f->waited_total + since;
7725f0c9c48SEd Cashin n /= USEC_PER_SEC;
773c450ba0fSEd Cashin if (aoe_deadsecs
774c450ba0fSEd Cashin && n > aoe_deadsecs
775c450ba0fSEd Cashin && !(f->flags & FFL_PROBE)) {
776896831f5SEd Cashin /* Waited too long. Device failure.
777896831f5SEd Cashin * Hang all frames on first hash bucket for downdev
778896831f5SEd Cashin * to clean up.
779896831f5SEd Cashin */
78064a80f5aSEd Cashin list_splice(&flist, &d->factive[0]);
7811da177e4SLinus Torvalds aoedev_downdev(d);
7823a0c40d2SEd Cashin goto out;
7831da177e4SLinus Torvalds }
78468e0d42fSEd L. Cashin
785896831f5SEd Cashin t = f->t;
786bbb44e30SEd Cashin n = f->waited + since;
787bbb44e30SEd Cashin n /= USEC_PER_SEC;
788bbb44e30SEd Cashin if (aoe_deadsecs && utgts > 0
789bbb44e30SEd Cashin && (n > aoe_deadsecs / utgts || n > HARD_SCORN_SECS))
790bbb44e30SEd Cashin scorn(t); /* avoid this target */
791d54d35acSEd Cashin
7923a0c40d2SEd Cashin if (t->maxout != 1) {
7933a0c40d2SEd Cashin t->ssthresh = t->maxout / 2;
7943a0c40d2SEd Cashin t->maxout = 1;
79568e0d42fSEd L. Cashin }
79668e0d42fSEd L. Cashin
797bbb44e30SEd Cashin if (f->flags & FFL_PROBE) {
798bbb44e30SEd Cashin t->nout_probes--;
799bbb44e30SEd Cashin } else {
80068e0d42fSEd L. Cashin ifp = getif(t, f->skb->dev);
80168e0d42fSEd L. Cashin if (ifp && ++ifp->lost > (t->nframes << 1)
80268e0d42fSEd L. Cashin && (ifp != t->ifs || t->ifs[1].nd)) {
80368e0d42fSEd L. Cashin ejectif(t, ifp);
80468e0d42fSEd L. Cashin ifp = NULL;
80568e0d42fSEd L. Cashin }
806bbb44e30SEd Cashin }
8073a0c40d2SEd Cashin list_move_tail(pos, &d->rexmitq);
8083a0c40d2SEd Cashin t->nout--;
8091da177e4SLinus Torvalds }
8103a0c40d2SEd Cashin rexmit_deferred(d);
81168e0d42fSEd L. Cashin
8123a0c40d2SEd Cashin out:
813bbb44e30SEd Cashin if ((d->flags & DEVFL_KICKME) && d->blkq) {
8144f51dc5eSEd L. Cashin d->flags &= ~DEVFL_KICKME;
8153582dd29SJens Axboe blk_mq_run_hw_queues(d->blkq, true);
8164f51dc5eSEd L. Cashin }
8171da177e4SLinus Torvalds
8181da177e4SLinus Torvalds d->timer.expires = jiffies + TIMERTICK;
8191da177e4SLinus Torvalds add_timer(&d->timer);
8201da177e4SLinus Torvalds
8211da177e4SLinus Torvalds spin_unlock_irqrestore(&d->lock, flags);
82269cf2d85SEd Cashin }
8231da177e4SLinus Torvalds
82469cf2d85SEd Cashin static void
bufinit(struct buf * buf,struct request * rq,struct bio * bio)82569cf2d85SEd Cashin bufinit(struct buf *buf, struct request *rq, struct bio *bio)
82669cf2d85SEd Cashin {
82769cf2d85SEd Cashin memset(buf, 0, sizeof(*buf));
82869cf2d85SEd Cashin buf->rq = rq;
82969cf2d85SEd Cashin buf->bio = bio;
830feb261e2SKent Overstreet buf->iter = bio->bi_iter;
83169cf2d85SEd Cashin }
83269cf2d85SEd Cashin
83369cf2d85SEd Cashin static struct buf *
nextbuf(struct aoedev * d)83469cf2d85SEd Cashin nextbuf(struct aoedev *d)
83569cf2d85SEd Cashin {
83669cf2d85SEd Cashin struct request *rq;
83769cf2d85SEd Cashin struct request_queue *q;
83861e7712eSChristoph Hellwig struct aoe_req *req;
83969cf2d85SEd Cashin struct buf *buf;
84069cf2d85SEd Cashin struct bio *bio;
84169cf2d85SEd Cashin
84269cf2d85SEd Cashin q = d->blkq;
84369cf2d85SEd Cashin if (q == NULL)
84469cf2d85SEd Cashin return NULL; /* initializing */
84569cf2d85SEd Cashin if (d->ip.buf)
84669cf2d85SEd Cashin return d->ip.buf;
84769cf2d85SEd Cashin rq = d->ip.rq;
84869cf2d85SEd Cashin if (rq == NULL) {
8493582dd29SJens Axboe rq = list_first_entry_or_null(&d->rq_list, struct request,
8503582dd29SJens Axboe queuelist);
85169cf2d85SEd Cashin if (rq == NULL)
85269cf2d85SEd Cashin return NULL;
8533582dd29SJens Axboe list_del_init(&rq->queuelist);
8543582dd29SJens Axboe blk_mq_start_request(rq);
85569cf2d85SEd Cashin d->ip.rq = rq;
85669cf2d85SEd Cashin d->ip.nxbio = rq->bio;
85761e7712eSChristoph Hellwig
85861e7712eSChristoph Hellwig req = blk_mq_rq_to_pdu(rq);
85961e7712eSChristoph Hellwig req->nr_bios = 0;
86061e7712eSChristoph Hellwig __rq_for_each_bio(bio, rq)
86161e7712eSChristoph Hellwig req->nr_bios++;
86269cf2d85SEd Cashin }
86369cf2d85SEd Cashin buf = mempool_alloc(d->bufpool, GFP_ATOMIC);
86469cf2d85SEd Cashin if (buf == NULL) {
86569cf2d85SEd Cashin pr_err("aoe: nextbuf: unable to mempool_alloc!\n");
86669cf2d85SEd Cashin return NULL;
86769cf2d85SEd Cashin }
86869cf2d85SEd Cashin bio = d->ip.nxbio;
86969cf2d85SEd Cashin bufinit(buf, rq, bio);
87069cf2d85SEd Cashin bio = bio->bi_next;
87169cf2d85SEd Cashin d->ip.nxbio = bio;
87269cf2d85SEd Cashin if (bio == NULL)
87369cf2d85SEd Cashin d->ip.rq = NULL;
87469cf2d85SEd Cashin return d->ip.buf = buf;
8751da177e4SLinus Torvalds }
8761da177e4SLinus Torvalds
87768e0d42fSEd L. Cashin /* enters with d->lock held */
87868e0d42fSEd L. Cashin void
aoecmd_work(struct aoedev * d)87968e0d42fSEd L. Cashin aoecmd_work(struct aoedev *d)
88068e0d42fSEd L. Cashin {
8813a0c40d2SEd Cashin rexmit_deferred(d);
88269cf2d85SEd Cashin while (aoecmd_ata_rw(d))
88369cf2d85SEd Cashin ;
88468e0d42fSEd L. Cashin }
88568e0d42fSEd L. Cashin
8863ae1c24eSEd L. Cashin /* this function performs work that has been deferred until sleeping is OK
8873ae1c24eSEd L. Cashin */
8883ae1c24eSEd L. Cashin void
aoecmd_sleepwork(struct work_struct * work)889c4028958SDavid Howells aoecmd_sleepwork(struct work_struct *work)
8903ae1c24eSEd L. Cashin {
891c4028958SDavid Howells struct aoedev *d = container_of(work, struct aoedev, work);
8923ae1c24eSEd L. Cashin
8933ae1c24eSEd L. Cashin if (d->flags & DEVFL_GDALLOC)
8943ae1c24eSEd L. Cashin aoeblk_gdalloc(d);
8953ae1c24eSEd L. Cashin
8963ae1c24eSEd L. Cashin if (d->flags & DEVFL_NEWSIZE) {
8978a6f7bbfSChristoph Hellwig set_capacity_and_notify(d->gd, d->ssize);
8988a6f7bbfSChristoph Hellwig
899b21faa25SEd Cashin spin_lock_irq(&d->lock);
9003ae1c24eSEd L. Cashin d->flags |= DEVFL_UP;
9013ae1c24eSEd L. Cashin d->flags &= ~DEVFL_NEWSIZE;
902b21faa25SEd Cashin spin_unlock_irq(&d->lock);
9033ae1c24eSEd L. Cashin }
9043ae1c24eSEd L. Cashin }
9053ae1c24eSEd L. Cashin
9061da177e4SLinus Torvalds static void
ata_ident_fixstring(u16 * id,int ns)907667be1e7SEd Cashin ata_ident_fixstring(u16 *id, int ns)
908667be1e7SEd Cashin {
909667be1e7SEd Cashin u16 s;
910667be1e7SEd Cashin
911667be1e7SEd Cashin while (ns-- > 0) {
912667be1e7SEd Cashin s = *id;
913667be1e7SEd Cashin *id++ = s >> 8 | s << 8;
914667be1e7SEd Cashin }
915667be1e7SEd Cashin }
916667be1e7SEd Cashin
917667be1e7SEd Cashin static void
ataid_complete(struct aoedev * d,struct aoetgt * t,unsigned char * id)91868e0d42fSEd L. Cashin ataid_complete(struct aoedev *d, struct aoetgt *t, unsigned char *id)
9191da177e4SLinus Torvalds {
9201da177e4SLinus Torvalds u64 ssize;
9211da177e4SLinus Torvalds u16 n;
9221da177e4SLinus Torvalds
9231da177e4SLinus Torvalds /* word 83: command set supported */
924f885f8d1SHarvey Harrison n = get_unaligned_le16(&id[83 << 1]);
9251da177e4SLinus Torvalds
9261da177e4SLinus Torvalds /* word 86: command set/feature enabled */
927f885f8d1SHarvey Harrison n |= get_unaligned_le16(&id[86 << 1]);
9281da177e4SLinus Torvalds
9291da177e4SLinus Torvalds if (n & (1<<10)) { /* bit 10: LBA 48 */
9301da177e4SLinus Torvalds d->flags |= DEVFL_EXT;
9311da177e4SLinus Torvalds
9321da177e4SLinus Torvalds /* word 100: number lba48 sectors */
933f885f8d1SHarvey Harrison ssize = get_unaligned_le64(&id[100 << 1]);
9341da177e4SLinus Torvalds
9351da177e4SLinus Torvalds /* set as in ide-disk.c:init_idedisk_capacity */
9361da177e4SLinus Torvalds d->geo.cylinders = ssize;
9371da177e4SLinus Torvalds d->geo.cylinders /= (255 * 63);
9381da177e4SLinus Torvalds d->geo.heads = 255;
9391da177e4SLinus Torvalds d->geo.sectors = 63;
9401da177e4SLinus Torvalds } else {
9411da177e4SLinus Torvalds d->flags &= ~DEVFL_EXT;
9421da177e4SLinus Torvalds
9431da177e4SLinus Torvalds /* number lba28 sectors */
944f885f8d1SHarvey Harrison ssize = get_unaligned_le32(&id[60 << 1]);
9451da177e4SLinus Torvalds
9461da177e4SLinus Torvalds /* NOTE: obsolete in ATA 6 */
947f885f8d1SHarvey Harrison d->geo.cylinders = get_unaligned_le16(&id[54 << 1]);
948f885f8d1SHarvey Harrison d->geo.heads = get_unaligned_le16(&id[55 << 1]);
949f885f8d1SHarvey Harrison d->geo.sectors = get_unaligned_le16(&id[56 << 1]);
9501da177e4SLinus Torvalds }
9513ae1c24eSEd L. Cashin
952667be1e7SEd Cashin ata_ident_fixstring((u16 *) &id[10<<1], 10); /* serial */
953667be1e7SEd Cashin ata_ident_fixstring((u16 *) &id[23<<1], 4); /* firmware */
954667be1e7SEd Cashin ata_ident_fixstring((u16 *) &id[27<<1], 20); /* model */
955667be1e7SEd Cashin memcpy(d->ident, id, sizeof(d->ident));
956667be1e7SEd Cashin
9573ae1c24eSEd L. Cashin if (d->ssize != ssize)
9581d75981aSEd L. Cashin printk(KERN_INFO
959411c41eeSHarvey Harrison "aoe: %pm e%ld.%d v%04x has %llu sectors\n",
960411c41eeSHarvey Harrison t->addr,
9613ae1c24eSEd L. Cashin d->aoemajor, d->aoeminor,
9623ae1c24eSEd L. Cashin d->fw_ver, (long long)ssize);
9631da177e4SLinus Torvalds d->ssize = ssize;
9641da177e4SLinus Torvalds d->geo.start = 0;
9656b9699bbSEd L. Cashin if (d->flags & (DEVFL_GDALLOC|DEVFL_NEWSIZE))
9666b9699bbSEd L. Cashin return;
9678a6f7bbfSChristoph Hellwig if (d->gd != NULL)
9683ae1c24eSEd L. Cashin d->flags |= DEVFL_NEWSIZE;
9698a6f7bbfSChristoph Hellwig else
9703ae1c24eSEd L. Cashin d->flags |= DEVFL_GDALLOC;
9710b8d7622STetsuo Handa queue_work(aoe_wq, &d->work);
9721da177e4SLinus Torvalds }
9731da177e4SLinus Torvalds
9741da177e4SLinus Torvalds static void
calc_rttavg(struct aoedev * d,struct aoetgt * t,int rtt)9753a0c40d2SEd Cashin calc_rttavg(struct aoedev *d, struct aoetgt *t, int rtt)
9761da177e4SLinus Torvalds {
9771da177e4SLinus Torvalds register long n;
9781da177e4SLinus Torvalds
9791da177e4SLinus Torvalds n = rtt;
9801da177e4SLinus Torvalds
9813a0c40d2SEd Cashin /* cf. Congestion Avoidance and Control, Jacobson & Karels, 1988 */
9823a0c40d2SEd Cashin n -= d->rttavg >> RTTSCALE;
9833a0c40d2SEd Cashin d->rttavg += n;
9843a0c40d2SEd Cashin if (n < 0)
9853a0c40d2SEd Cashin n = -n;
9863a0c40d2SEd Cashin n -= d->rttdev >> RTTDSCALE;
9873a0c40d2SEd Cashin d->rttdev += n;
9883a0c40d2SEd Cashin
9893a0c40d2SEd Cashin if (!t || t->maxout >= t->nframes)
9903a0c40d2SEd Cashin return;
9913a0c40d2SEd Cashin if (t->maxout < t->ssthresh)
9923a0c40d2SEd Cashin t->maxout += 1;
9933a0c40d2SEd Cashin else if (t->nout == t->maxout && t->next_cwnd-- == 0) {
9943a0c40d2SEd Cashin t->maxout += 1;
9953a0c40d2SEd Cashin t->next_cwnd = t->maxout;
9963a0c40d2SEd Cashin }
9971da177e4SLinus Torvalds }
9981da177e4SLinus Torvalds
99968e0d42fSEd L. Cashin static struct aoetgt *
gettgt(struct aoedev * d,char * addr)100068e0d42fSEd L. Cashin gettgt(struct aoedev *d, char *addr)
100168e0d42fSEd L. Cashin {
100268e0d42fSEd L. Cashin struct aoetgt **t, **e;
100368e0d42fSEd L. Cashin
100468e0d42fSEd L. Cashin t = d->targets;
100571114ec4SEd Cashin e = t + d->ntargets;
100668e0d42fSEd L. Cashin for (; t < e && *t; t++)
100768e0d42fSEd L. Cashin if (memcmp((*t)->addr, addr, sizeof((*t)->addr)) == 0)
100868e0d42fSEd L. Cashin return *t;
100968e0d42fSEd L. Cashin return NULL;
101068e0d42fSEd L. Cashin }
101168e0d42fSEd L. Cashin
10123d5b0605SEd Cashin static void
bvcpy(struct sk_buff * skb,struct bio * bio,struct bvec_iter iter,long cnt)1013feb261e2SKent Overstreet bvcpy(struct sk_buff *skb, struct bio *bio, struct bvec_iter iter, long cnt)
10143d5b0605SEd Cashin {
10153d5b0605SEd Cashin int soff = 0;
1016feb261e2SKent Overstreet struct bio_vec bv;
1017feb261e2SKent Overstreet
1018feb261e2SKent Overstreet iter.bi_size = cnt;
1019feb261e2SKent Overstreet
1020feb261e2SKent Overstreet __bio_for_each_segment(bv, bio, iter, iter) {
1021b7ab4611SChristoph Hellwig char *p = bvec_kmap_local(&bv);
1022feb261e2SKent Overstreet skb_copy_bits(skb, soff, p, bv.bv_len);
1023b7ab4611SChristoph Hellwig kunmap_local(p);
1024feb261e2SKent Overstreet soff += bv.bv_len;
1025feb261e2SKent Overstreet }
10263d5b0605SEd Cashin }
10273d5b0605SEd Cashin
102869cf2d85SEd Cashin void
aoe_end_request(struct aoedev * d,struct request * rq,int fastfail)102969cf2d85SEd Cashin aoe_end_request(struct aoedev *d, struct request *rq, int fastfail)
103069cf2d85SEd Cashin {
103169cf2d85SEd Cashin struct bio *bio;
103269cf2d85SEd Cashin int bok;
103369cf2d85SEd Cashin struct request_queue *q;
10343582dd29SJens Axboe blk_status_t err = BLK_STS_OK;
103569cf2d85SEd Cashin
103669cf2d85SEd Cashin q = d->blkq;
103769cf2d85SEd Cashin if (rq == d->ip.rq)
103869cf2d85SEd Cashin d->ip.rq = NULL;
103969cf2d85SEd Cashin do {
104069cf2d85SEd Cashin bio = rq->bio;
10414e4cbee9SChristoph Hellwig bok = !fastfail && !bio->bi_status;
10423582dd29SJens Axboe if (!bok)
10433582dd29SJens Axboe err = BLK_STS_IOERR;
10443582dd29SJens Axboe } while (blk_update_request(rq, bok ? BLK_STS_OK : BLK_STS_IOERR, bio->bi_iter.bi_size));
10453582dd29SJens Axboe
10463582dd29SJens Axboe __blk_mq_end_request(rq, err);
104769cf2d85SEd Cashin
104859788683SKees Cook /* cf. https://lore.kernel.org/lkml/20061031071040.GS14055@kernel.dk/ */
104969cf2d85SEd Cashin if (!fastfail)
10503582dd29SJens Axboe blk_mq_run_hw_queues(q, true);
105169cf2d85SEd Cashin }
105269cf2d85SEd Cashin
105369cf2d85SEd Cashin static void
aoe_end_buf(struct aoedev * d,struct buf * buf)105469cf2d85SEd Cashin aoe_end_buf(struct aoedev *d, struct buf *buf)
105569cf2d85SEd Cashin {
105661e7712eSChristoph Hellwig struct request *rq = buf->rq;
105761e7712eSChristoph Hellwig struct aoe_req *req = blk_mq_rq_to_pdu(rq);
105869cf2d85SEd Cashin
105969cf2d85SEd Cashin if (buf == d->ip.buf)
106069cf2d85SEd Cashin d->ip.buf = NULL;
106169cf2d85SEd Cashin mempool_free(buf, d->bufpool);
106261e7712eSChristoph Hellwig if (--req->nr_bios == 0)
106369cf2d85SEd Cashin aoe_end_request(d, rq, 0);
106469cf2d85SEd Cashin }
106569cf2d85SEd Cashin
10663d5b0605SEd Cashin static void
ktiocomplete(struct frame * f)1067896831f5SEd Cashin ktiocomplete(struct frame *f)
10683d5b0605SEd Cashin {
1069ddec63e8SEd L. Cashin struct aoe_hdr *hin, *hout;
10701da177e4SLinus Torvalds struct aoe_atahdr *ahin, *ahout;
10711da177e4SLinus Torvalds struct buf *buf;
1072896831f5SEd Cashin struct sk_buff *skb;
107368e0d42fSEd L. Cashin struct aoetgt *t;
107468e0d42fSEd L. Cashin struct aoeif *ifp;
1075896831f5SEd Cashin struct aoedev *d;
1076896831f5SEd Cashin long n;
1077bbb44e30SEd Cashin int untainted;
1078896831f5SEd Cashin
1079896831f5SEd Cashin if (f == NULL)
1080896831f5SEd Cashin return;
1081896831f5SEd Cashin
1082896831f5SEd Cashin t = f->t;
1083896831f5SEd Cashin d = t->d;
1084bbb44e30SEd Cashin skb = f->r_skb;
1085bbb44e30SEd Cashin buf = f->buf;
1086bbb44e30SEd Cashin if (f->flags & FFL_PROBE)
1087bbb44e30SEd Cashin goto out;
1088bbb44e30SEd Cashin if (!skb) /* just fail the buf. */
1089bbb44e30SEd Cashin goto noskb;
1090896831f5SEd Cashin
1091896831f5SEd Cashin hout = (struct aoe_hdr *) skb_mac_header(f->skb);
1092896831f5SEd Cashin ahout = (struct aoe_atahdr *) (hout+1);
1093896831f5SEd Cashin
1094896831f5SEd Cashin hin = (struct aoe_hdr *) skb->data;
1095896831f5SEd Cashin skb_pull(skb, sizeof(*hin));
1096896831f5SEd Cashin ahin = (struct aoe_atahdr *) skb->data;
1097896831f5SEd Cashin skb_pull(skb, sizeof(*ahin));
1098896831f5SEd Cashin if (ahin->cmdstat & 0xa9) { /* these bits cleared on success */
1099896831f5SEd Cashin pr_err("aoe: ata error cmd=%2.2Xh stat=%2.2Xh from e%ld.%d\n",
1100896831f5SEd Cashin ahout->cmdstat, ahin->cmdstat,
1101896831f5SEd Cashin d->aoemajor, d->aoeminor);
1102896831f5SEd Cashin noskb: if (buf)
11034e4cbee9SChristoph Hellwig buf->bio->bi_status = BLK_STS_IOERR;
1104bbb44e30SEd Cashin goto out;
1105896831f5SEd Cashin }
1106896831f5SEd Cashin
1107896831f5SEd Cashin n = ahout->scnt << 9;
1108896831f5SEd Cashin switch (ahout->cmdstat) {
1109896831f5SEd Cashin case ATA_CMD_PIO_READ:
1110896831f5SEd Cashin case ATA_CMD_PIO_READ_EXT:
1111896831f5SEd Cashin if (skb->len < n) {
1112bf29754aSEd Cashin pr_err("%s e%ld.%d. skb->len=%d need=%ld\n",
1113bf29754aSEd Cashin "aoe: runt data size in read from",
1114bf29754aSEd Cashin (long) d->aoemajor, d->aoeminor,
1115896831f5SEd Cashin skb->len, n);
11164e4cbee9SChristoph Hellwig buf->bio->bi_status = BLK_STS_IOERR;
1117896831f5SEd Cashin break;
1118896831f5SEd Cashin }
1119feb261e2SKent Overstreet if (n > f->iter.bi_size) {
1120feb261e2SKent Overstreet pr_err_ratelimited("%s e%ld.%d. bytes=%ld need=%u\n",
1121feb261e2SKent Overstreet "aoe: too-large data size in read from",
1122feb261e2SKent Overstreet (long) d->aoemajor, d->aoeminor,
1123feb261e2SKent Overstreet n, f->iter.bi_size);
11244e4cbee9SChristoph Hellwig buf->bio->bi_status = BLK_STS_IOERR;
1125feb261e2SKent Overstreet break;
1126feb261e2SKent Overstreet }
1127feb261e2SKent Overstreet bvcpy(skb, f->buf->bio, f->iter, n);
1128df561f66SGustavo A. R. Silva fallthrough;
1129896831f5SEd Cashin case ATA_CMD_PIO_WRITE:
1130896831f5SEd Cashin case ATA_CMD_PIO_WRITE_EXT:
1131896831f5SEd Cashin spin_lock_irq(&d->lock);
1132896831f5SEd Cashin ifp = getif(t, skb->dev);
11333f0f0133SEd Cashin if (ifp)
1134896831f5SEd Cashin ifp->lost = 0;
1135896831f5SEd Cashin spin_unlock_irq(&d->lock);
1136896831f5SEd Cashin break;
1137896831f5SEd Cashin case ATA_CMD_ID_ATA:
1138896831f5SEd Cashin if (skb->len < 512) {
1139bf29754aSEd Cashin pr_info("%s e%ld.%d. skb->len=%d need=512\n",
1140bf29754aSEd Cashin "aoe: runt data size in ataid from",
1141bf29754aSEd Cashin (long) d->aoemajor, d->aoeminor,
1142896831f5SEd Cashin skb->len);
1143896831f5SEd Cashin break;
1144896831f5SEd Cashin }
1145896831f5SEd Cashin if (skb_linearize(skb))
1146896831f5SEd Cashin break;
1147896831f5SEd Cashin spin_lock_irq(&d->lock);
1148896831f5SEd Cashin ataid_complete(d, t, skb->data);
1149896831f5SEd Cashin spin_unlock_irq(&d->lock);
1150896831f5SEd Cashin break;
1151896831f5SEd Cashin default:
1152896831f5SEd Cashin pr_info("aoe: unrecognized ata command %2.2Xh for %d.%d\n",
1153896831f5SEd Cashin ahout->cmdstat,
1154896831f5SEd Cashin be16_to_cpu(get_unaligned(&hin->major)),
1155896831f5SEd Cashin hin->minor);
1156896831f5SEd Cashin }
1157bbb44e30SEd Cashin out:
1158896831f5SEd Cashin spin_lock_irq(&d->lock);
1159bbb44e30SEd Cashin if (t->taint > 0
1160bbb44e30SEd Cashin && --t->taint > 0
1161bbb44e30SEd Cashin && t->nout_probes == 0) {
1162bbb44e30SEd Cashin count_targets(d, &untainted);
1163bbb44e30SEd Cashin if (untainted > 0) {
1164bbb44e30SEd Cashin probe(t);
1165bbb44e30SEd Cashin t->nout_probes++;
1166bbb44e30SEd Cashin }
1167bbb44e30SEd Cashin }
1168896831f5SEd Cashin
1169896831f5SEd Cashin aoe_freetframe(f);
1170896831f5SEd Cashin
1171feb261e2SKent Overstreet if (buf && --buf->nframesout == 0 && buf->iter.bi_size == 0)
117269cf2d85SEd Cashin aoe_end_buf(d, buf);
1173896831f5SEd Cashin
1174896831f5SEd Cashin spin_unlock_irq(&d->lock);
117569cf2d85SEd Cashin aoedev_put(d);
1176896831f5SEd Cashin dev_kfree_skb(skb);
1177896831f5SEd Cashin }
1178896831f5SEd Cashin
1179896831f5SEd Cashin /* Enters with iocq.lock held.
1180896831f5SEd Cashin * Returns true iff responses needing processing remain.
1181896831f5SEd Cashin */
1182896831f5SEd Cashin static int
ktio(int id)11838030d343SEd Cashin ktio(int id)
1184896831f5SEd Cashin {
1185896831f5SEd Cashin struct frame *f;
1186896831f5SEd Cashin struct list_head *pos;
1187896831f5SEd Cashin int i;
11888030d343SEd Cashin int actual_id;
1189896831f5SEd Cashin
1190896831f5SEd Cashin for (i = 0; ; ++i) {
1191896831f5SEd Cashin if (i == MAXIOC)
1192896831f5SEd Cashin return 1;
11938030d343SEd Cashin if (list_empty(&iocq[id].head))
1194896831f5SEd Cashin return 0;
11958030d343SEd Cashin pos = iocq[id].head.next;
1196896831f5SEd Cashin list_del(pos);
1197896831f5SEd Cashin f = list_entry(pos, struct frame, head);
11988030d343SEd Cashin spin_unlock_irq(&iocq[id].lock);
1199896831f5SEd Cashin ktiocomplete(f);
12008030d343SEd Cashin
12018030d343SEd Cashin /* Figure out if extra threads are required. */
12028030d343SEd Cashin actual_id = f->t->d->aoeminor % ncpus;
12038030d343SEd Cashin
12048030d343SEd Cashin if (!kts[actual_id].active) {
12058030d343SEd Cashin BUG_ON(id != 0);
12068030d343SEd Cashin mutex_lock(&ktio_spawn_lock);
12078030d343SEd Cashin if (!kts[actual_id].active
12088030d343SEd Cashin && aoe_ktstart(&kts[actual_id]) == 0)
12098030d343SEd Cashin kts[actual_id].active = 1;
12108030d343SEd Cashin mutex_unlock(&ktio_spawn_lock);
12118030d343SEd Cashin }
12128030d343SEd Cashin spin_lock_irq(&iocq[id].lock);
1213896831f5SEd Cashin }
1214896831f5SEd Cashin }
1215896831f5SEd Cashin
1216896831f5SEd Cashin static int
kthread(void * vp)1217896831f5SEd Cashin kthread(void *vp)
1218896831f5SEd Cashin {
1219896831f5SEd Cashin struct ktstate *k;
1220896831f5SEd Cashin DECLARE_WAITQUEUE(wait, current);
1221896831f5SEd Cashin int more;
1222896831f5SEd Cashin
1223896831f5SEd Cashin k = vp;
1224896831f5SEd Cashin current->flags |= PF_NOFREEZE;
1225896831f5SEd Cashin set_user_nice(current, -10);
1226896831f5SEd Cashin complete(&k->rendez); /* tell spawner we're running */
1227896831f5SEd Cashin do {
1228896831f5SEd Cashin spin_lock_irq(k->lock);
12298030d343SEd Cashin more = k->fn(k->id);
1230896831f5SEd Cashin if (!more) {
1231896831f5SEd Cashin add_wait_queue(k->waitq, &wait);
1232896831f5SEd Cashin __set_current_state(TASK_INTERRUPTIBLE);
1233896831f5SEd Cashin }
1234896831f5SEd Cashin spin_unlock_irq(k->lock);
1235896831f5SEd Cashin if (!more) {
1236896831f5SEd Cashin schedule();
1237896831f5SEd Cashin remove_wait_queue(k->waitq, &wait);
1238896831f5SEd Cashin } else
1239896831f5SEd Cashin cond_resched();
1240896831f5SEd Cashin } while (!kthread_should_stop());
1241896831f5SEd Cashin complete(&k->rendez); /* tell spawner we're stopping */
1242896831f5SEd Cashin return 0;
1243896831f5SEd Cashin }
1244896831f5SEd Cashin
1245eb086ec5SEd Cashin void
aoe_ktstop(struct ktstate * k)1246896831f5SEd Cashin aoe_ktstop(struct ktstate *k)
1247896831f5SEd Cashin {
1248896831f5SEd Cashin kthread_stop(k->task);
1249896831f5SEd Cashin wait_for_completion(&k->rendez);
1250896831f5SEd Cashin }
1251896831f5SEd Cashin
1252eb086ec5SEd Cashin int
aoe_ktstart(struct ktstate * k)1253896831f5SEd Cashin aoe_ktstart(struct ktstate *k)
1254896831f5SEd Cashin {
1255896831f5SEd Cashin struct task_struct *task;
1256896831f5SEd Cashin
1257896831f5SEd Cashin init_completion(&k->rendez);
1258f170168bSKees Cook task = kthread_run(kthread, k, "%s", k->name);
1259896831f5SEd Cashin if (task == NULL || IS_ERR(task))
1260896831f5SEd Cashin return -ENOMEM;
1261896831f5SEd Cashin k->task = task;
1262896831f5SEd Cashin wait_for_completion(&k->rendez); /* allow kthread to start */
1263896831f5SEd Cashin init_completion(&k->rendez); /* for waiting for exit later */
1264896831f5SEd Cashin return 0;
1265896831f5SEd Cashin }
1266896831f5SEd Cashin
1267896831f5SEd Cashin /* pass it off to kthreads for processing */
1268896831f5SEd Cashin static void
ktcomplete(struct frame * f,struct sk_buff * skb)1269896831f5SEd Cashin ktcomplete(struct frame *f, struct sk_buff *skb)
1270896831f5SEd Cashin {
12718030d343SEd Cashin int id;
1272896831f5SEd Cashin ulong flags;
1273896831f5SEd Cashin
1274896831f5SEd Cashin f->r_skb = skb;
12758030d343SEd Cashin id = f->t->d->aoeminor % ncpus;
12768030d343SEd Cashin spin_lock_irqsave(&iocq[id].lock, flags);
12778030d343SEd Cashin if (!kts[id].active) {
12788030d343SEd Cashin spin_unlock_irqrestore(&iocq[id].lock, flags);
12798030d343SEd Cashin /* The thread with id has not been spawned yet,
12808030d343SEd Cashin * so delegate the work to the main thread and
12818030d343SEd Cashin * try spawning a new thread.
12828030d343SEd Cashin */
12838030d343SEd Cashin id = 0;
12848030d343SEd Cashin spin_lock_irqsave(&iocq[id].lock, flags);
12858030d343SEd Cashin }
12868030d343SEd Cashin list_add_tail(&f->head, &iocq[id].head);
12878030d343SEd Cashin spin_unlock_irqrestore(&iocq[id].lock, flags);
12888030d343SEd Cashin wake_up(&ktiowq[id]);
1289896831f5SEd Cashin }
1290896831f5SEd Cashin
1291896831f5SEd Cashin struct sk_buff *
aoecmd_ata_rsp(struct sk_buff * skb)1292896831f5SEd Cashin aoecmd_ata_rsp(struct sk_buff *skb)
1293896831f5SEd Cashin {
1294896831f5SEd Cashin struct aoedev *d;
1295896831f5SEd Cashin struct aoe_hdr *h;
1296896831f5SEd Cashin struct frame *f;
1297896831f5SEd Cashin u32 n;
12981da177e4SLinus Torvalds ulong flags;
12991da177e4SLinus Torvalds char ebuf[128];
130032465c65Secashin@coraid.com u16 aoemajor;
13011da177e4SLinus Torvalds
1302896831f5SEd Cashin h = (struct aoe_hdr *) skb->data;
1303896831f5SEd Cashin aoemajor = be16_to_cpu(get_unaligned(&h->major));
13040c966214SEd Cashin d = aoedev_by_aoeaddr(aoemajor, h->minor, 0);
13051da177e4SLinus Torvalds if (d == NULL) {
13061da177e4SLinus Torvalds snprintf(ebuf, sizeof ebuf, "aoecmd_ata_rsp: ata response "
13071da177e4SLinus Torvalds "for unknown device %d.%d\n",
1308896831f5SEd Cashin aoemajor, h->minor);
13091da177e4SLinus Torvalds aoechr_error(ebuf);
1310896831f5SEd Cashin return skb;
13111da177e4SLinus Torvalds }
13121da177e4SLinus Torvalds
13131da177e4SLinus Torvalds spin_lock_irqsave(&d->lock, flags);
13141da177e4SLinus Torvalds
1315896831f5SEd Cashin n = be32_to_cpu(get_unaligned(&h->tag));
131664a80f5aSEd Cashin f = getframe(d, n);
13173a0c40d2SEd Cashin if (f) {
13185f0c9c48SEd Cashin calc_rttavg(d, f->t, tsince_hr(f));
13193a0c40d2SEd Cashin f->t->nout--;
1320bbb44e30SEd Cashin if (f->flags & FFL_PROBE)
1321bbb44e30SEd Cashin f->t->nout_probes--;
13223a0c40d2SEd Cashin } else {
13233a0c40d2SEd Cashin f = getframe_deferred(d, n);
13243a0c40d2SEd Cashin if (f) {
13255f0c9c48SEd Cashin calc_rttavg(d, NULL, tsince_hr(f));
13263a0c40d2SEd Cashin } else {
13273a0c40d2SEd Cashin calc_rttavg(d, NULL, tsince(n));
13281da177e4SLinus Torvalds spin_unlock_irqrestore(&d->lock, flags);
132969cf2d85SEd Cashin aoedev_put(d);
13303a0c40d2SEd Cashin snprintf(ebuf, sizeof(ebuf),
13312292a7e1SEd Cashin "%15s e%d.%d tag=%08x@%08lx s=%pm d=%pm\n",
13321da177e4SLinus Torvalds "unexpected rsp",
1333896831f5SEd Cashin get_unaligned_be16(&h->major),
1334896831f5SEd Cashin h->minor,
1335896831f5SEd Cashin get_unaligned_be32(&h->tag),
13362292a7e1SEd Cashin jiffies,
13372292a7e1SEd Cashin h->src,
13382292a7e1SEd Cashin h->dst);
13391da177e4SLinus Torvalds aoechr_error(ebuf);
1340896831f5SEd Cashin return skb;
13411da177e4SLinus Torvalds }
13423a0c40d2SEd Cashin }
13431da177e4SLinus Torvalds aoecmd_work(d);
13441da177e4SLinus Torvalds
13451da177e4SLinus Torvalds spin_unlock_irqrestore(&d->lock, flags);
1346896831f5SEd Cashin
1347896831f5SEd Cashin ktcomplete(f, skb);
1348896831f5SEd Cashin
1349896831f5SEd Cashin /*
1350896831f5SEd Cashin * Note here that we do not perform an aoedev_put, as we are
1351896831f5SEd Cashin * leaving this reference for the ktio to release.
1352896831f5SEd Cashin */
1353896831f5SEd Cashin return NULL;
13541da177e4SLinus Torvalds }
13551da177e4SLinus Torvalds
13561da177e4SLinus Torvalds void
aoecmd_cfg(ushort aoemajor,unsigned char aoeminor)13571da177e4SLinus Torvalds aoecmd_cfg(ushort aoemajor, unsigned char aoeminor)
13581da177e4SLinus Torvalds {
1359e9bb8fb0SDavid S. Miller struct sk_buff_head queue;
13601da177e4SLinus Torvalds
1361e9bb8fb0SDavid S. Miller __skb_queue_head_init(&queue);
1362e9bb8fb0SDavid S. Miller aoecmd_cfg_pkts(aoemajor, aoeminor, &queue);
1363e9bb8fb0SDavid S. Miller aoenet_xmit(&queue);
13641da177e4SLinus Torvalds }
13651da177e4SLinus Torvalds
136668e0d42fSEd L. Cashin struct sk_buff *
aoecmd_ata_id(struct aoedev * d)13671da177e4SLinus Torvalds aoecmd_ata_id(struct aoedev *d)
13681da177e4SLinus Torvalds {
13691da177e4SLinus Torvalds struct aoe_hdr *h;
13701da177e4SLinus Torvalds struct aoe_atahdr *ah;
13711da177e4SLinus Torvalds struct frame *f;
13721da177e4SLinus Torvalds struct sk_buff *skb;
137368e0d42fSEd L. Cashin struct aoetgt *t;
13741da177e4SLinus Torvalds
1375896831f5SEd Cashin f = newframe(d);
137668e0d42fSEd L. Cashin if (f == NULL)
13771da177e4SLinus Torvalds return NULL;
137868e0d42fSEd L. Cashin
137968e0d42fSEd L. Cashin t = *d->tgt;
13801da177e4SLinus Torvalds
13811da177e4SLinus Torvalds /* initialize the headers & frame */
1382e407a7f6SEd L. Cashin skb = f->skb;
1383abdbf94dSEd L. Cashin h = (struct aoe_hdr *) skb_mac_header(skb);
13841da177e4SLinus Torvalds ah = (struct aoe_atahdr *) (h+1);
138519900cdeSEd L. Cashin skb_put(skb, sizeof *h + sizeof *ah);
138619900cdeSEd L. Cashin memset(h, 0, skb->len);
138768e0d42fSEd L. Cashin f->tag = aoehdr_atainit(d, t, h);
1388896831f5SEd Cashin fhash(f);
138968e0d42fSEd L. Cashin t->nout++;
13901da177e4SLinus Torvalds f->waited = 0;
13913fc9b032SEd Cashin f->waited_total = 0;
13921da177e4SLinus Torvalds
13931da177e4SLinus Torvalds /* set up ata header */
13941da177e4SLinus Torvalds ah->scnt = 1;
139504b3ab52SBartlomiej Zolnierkiewicz ah->cmdstat = ATA_CMD_ID_ATA;
13961da177e4SLinus Torvalds ah->lba3 = 0xa0;
13971da177e4SLinus Torvalds
139868e0d42fSEd L. Cashin skb->dev = t->ifp->nd;
13991da177e4SLinus Torvalds
14003a0c40d2SEd Cashin d->rttavg = RTTAVG_INIT;
14013a0c40d2SEd Cashin d->rttdev = RTTDEV_INIT;
1402841b86f3SKees Cook d->timer.function = rexmit_timer;
14031da177e4SLinus Torvalds
14045f0c9c48SEd Cashin skb = skb_clone(skb, GFP_ATOMIC);
140585cf955dSTina Ruchandani if (skb)
140685cf955dSTina Ruchandani f->sent = ktime_get();
14075f0c9c48SEd Cashin
14085f0c9c48SEd Cashin return skb;
14091da177e4SLinus Torvalds }
14101da177e4SLinus Torvalds
141171114ec4SEd Cashin static struct aoetgt **
grow_targets(struct aoedev * d)141271114ec4SEd Cashin grow_targets(struct aoedev *d)
141371114ec4SEd Cashin {
141471114ec4SEd Cashin ulong oldn, newn;
141571114ec4SEd Cashin struct aoetgt **tt;
141671114ec4SEd Cashin
141771114ec4SEd Cashin oldn = d->ntargets;
141871114ec4SEd Cashin newn = oldn * 2;
141971114ec4SEd Cashin tt = kcalloc(newn, sizeof(*d->targets), GFP_ATOMIC);
142071114ec4SEd Cashin if (!tt)
142171114ec4SEd Cashin return NULL;
142271114ec4SEd Cashin memmove(tt, d->targets, sizeof(*d->targets) * oldn);
142371114ec4SEd Cashin d->tgt = tt + (d->tgt - d->targets);
142471114ec4SEd Cashin kfree(d->targets);
142571114ec4SEd Cashin d->targets = tt;
142671114ec4SEd Cashin d->ntargets = newn;
142771114ec4SEd Cashin
142871114ec4SEd Cashin return &d->targets[oldn];
142971114ec4SEd Cashin }
143071114ec4SEd Cashin
143168e0d42fSEd L. Cashin static struct aoetgt *
addtgt(struct aoedev * d,char * addr,ulong nframes)143268e0d42fSEd L. Cashin addtgt(struct aoedev *d, char *addr, ulong nframes)
143368e0d42fSEd L. Cashin {
143468e0d42fSEd L. Cashin struct aoetgt *t, **tt, **te;
143568e0d42fSEd L. Cashin
143668e0d42fSEd L. Cashin tt = d->targets;
143771114ec4SEd Cashin te = tt + d->ntargets;
143868e0d42fSEd L. Cashin for (; tt < te && *tt; tt++)
143968e0d42fSEd L. Cashin ;
144068e0d42fSEd L. Cashin
1441578c4aa0SEd L. Cashin if (tt == te) {
144271114ec4SEd Cashin tt = grow_targets(d);
144371114ec4SEd Cashin if (!tt)
144471114ec4SEd Cashin goto nomem;
1445578c4aa0SEd L. Cashin }
1446896831f5SEd Cashin t = kzalloc(sizeof(*t), GFP_ATOMIC);
144771114ec4SEd Cashin if (!t)
144871114ec4SEd Cashin goto nomem;
144968e0d42fSEd L. Cashin t->nframes = nframes;
1450896831f5SEd Cashin t->d = d;
145168e0d42fSEd L. Cashin memcpy(t->addr, addr, sizeof t->addr);
145268e0d42fSEd L. Cashin t->ifp = t->ifs;
14533a0c40d2SEd Cashin aoecmd_wreset(t);
1454bbb44e30SEd Cashin t->maxout = t->nframes / 2;
1455896831f5SEd Cashin INIT_LIST_HEAD(&t->ffree);
145668e0d42fSEd L. Cashin return *tt = t;
145771114ec4SEd Cashin
145871114ec4SEd Cashin nomem:
145971114ec4SEd Cashin pr_info("aoe: cannot allocate memory to add target\n");
146071114ec4SEd Cashin return NULL;
146168e0d42fSEd L. Cashin }
146268e0d42fSEd L. Cashin
14633f0f0133SEd Cashin static void
setdbcnt(struct aoedev * d)14643f0f0133SEd Cashin setdbcnt(struct aoedev *d)
14653f0f0133SEd Cashin {
14663f0f0133SEd Cashin struct aoetgt **t, **e;
14673f0f0133SEd Cashin int bcnt = 0;
14683f0f0133SEd Cashin
14693f0f0133SEd Cashin t = d->targets;
147071114ec4SEd Cashin e = t + d->ntargets;
14713f0f0133SEd Cashin for (; t < e && *t; t++)
14723f0f0133SEd Cashin if (bcnt == 0 || bcnt > (*t)->minbcnt)
14733f0f0133SEd Cashin bcnt = (*t)->minbcnt;
14743f0f0133SEd Cashin if (bcnt != d->maxbcnt) {
14753f0f0133SEd Cashin d->maxbcnt = bcnt;
14763f0f0133SEd Cashin pr_info("aoe: e%ld.%d: setting %d byte data frames\n",
14773f0f0133SEd Cashin d->aoemajor, d->aoeminor, bcnt);
14783f0f0133SEd Cashin }
14793f0f0133SEd Cashin }
14803f0f0133SEd Cashin
14813f0f0133SEd Cashin static void
setifbcnt(struct aoetgt * t,struct net_device * nd,int bcnt)14823f0f0133SEd Cashin setifbcnt(struct aoetgt *t, struct net_device *nd, int bcnt)
14833f0f0133SEd Cashin {
14843f0f0133SEd Cashin struct aoedev *d;
14853f0f0133SEd Cashin struct aoeif *p, *e;
14863f0f0133SEd Cashin int minbcnt;
14873f0f0133SEd Cashin
14883f0f0133SEd Cashin d = t->d;
14893f0f0133SEd Cashin minbcnt = bcnt;
14903f0f0133SEd Cashin p = t->ifs;
14913f0f0133SEd Cashin e = p + NAOEIFS;
14923f0f0133SEd Cashin for (; p < e; p++) {
14933f0f0133SEd Cashin if (p->nd == NULL)
14943f0f0133SEd Cashin break; /* end of the valid interfaces */
14953f0f0133SEd Cashin if (p->nd == nd) {
14963f0f0133SEd Cashin p->bcnt = bcnt; /* we're updating */
14973f0f0133SEd Cashin nd = NULL;
14983f0f0133SEd Cashin } else if (minbcnt > p->bcnt)
14993f0f0133SEd Cashin minbcnt = p->bcnt; /* find the min interface */
15003f0f0133SEd Cashin }
15013f0f0133SEd Cashin if (nd) {
15023f0f0133SEd Cashin if (p == e) {
15033f0f0133SEd Cashin pr_err("aoe: device setifbcnt failure; too many interfaces.\n");
15043f0f0133SEd Cashin return;
15053f0f0133SEd Cashin }
15061b86fda9SEd Cashin dev_hold(nd);
15073f0f0133SEd Cashin p->nd = nd;
15083f0f0133SEd Cashin p->bcnt = bcnt;
15093f0f0133SEd Cashin }
15103f0f0133SEd Cashin t->minbcnt = minbcnt;
15113f0f0133SEd Cashin setdbcnt(d);
15123f0f0133SEd Cashin }
15133f0f0133SEd Cashin
15141da177e4SLinus Torvalds void
aoecmd_cfg_rsp(struct sk_buff * skb)15151da177e4SLinus Torvalds aoecmd_cfg_rsp(struct sk_buff *skb)
15161da177e4SLinus Torvalds {
15171da177e4SLinus Torvalds struct aoedev *d;
15181da177e4SLinus Torvalds struct aoe_hdr *h;
15191da177e4SLinus Torvalds struct aoe_cfghdr *ch;
152068e0d42fSEd L. Cashin struct aoetgt *t;
15210c966214SEd Cashin ulong flags, aoemajor;
15221da177e4SLinus Torvalds struct sk_buff *sl;
152369cf2d85SEd Cashin struct sk_buff_head queue;
152419bf2635SEd L. Cashin u16 n;
15251da177e4SLinus Torvalds
152669cf2d85SEd Cashin sl = NULL;
1527abdbf94dSEd L. Cashin h = (struct aoe_hdr *) skb_mac_header(skb);
15281da177e4SLinus Torvalds ch = (struct aoe_cfghdr *) (h+1);
15291da177e4SLinus Torvalds
15301da177e4SLinus Torvalds /*
15311da177e4SLinus Torvalds * Enough people have their dip switches set backwards to
15321da177e4SLinus Torvalds * warrant a loud message for this special case.
15331da177e4SLinus Torvalds */
1534823ed72eSHarvey Harrison aoemajor = get_unaligned_be16(&h->major);
15351da177e4SLinus Torvalds if (aoemajor == 0xfff) {
1536a12c93f0SEd L. Cashin printk(KERN_ERR "aoe: Warning: shelf address is all ones. "
15376bb6285fSEd L. Cashin "Check shelf dip switches.\n");
15381da177e4SLinus Torvalds return;
15391da177e4SLinus Torvalds }
15407159e969SEd Cashin if (aoemajor == 0xffff) {
15417159e969SEd Cashin pr_info("aoe: e%ld.%d: broadcast shelf number invalid\n",
15420c966214SEd Cashin aoemajor, (int) h->minor);
15436583303cSEd Cashin return;
15446583303cSEd Cashin }
15457159e969SEd Cashin if (h->minor == 0xff) {
15467159e969SEd Cashin pr_info("aoe: e%ld.%d: broadcast slot number invalid\n",
15477159e969SEd Cashin aoemajor, (int) h->minor);
15481da177e4SLinus Torvalds return;
15491da177e4SLinus Torvalds }
15501da177e4SLinus Torvalds
155119bf2635SEd L. Cashin n = be16_to_cpu(ch->bufcnt);
15527df620d8SEd L. Cashin if (n > aoe_maxout) /* keep it reasonable */
15537df620d8SEd L. Cashin n = aoe_maxout;
15541da177e4SLinus Torvalds
15557159e969SEd Cashin d = aoedev_by_aoeaddr(aoemajor, h->minor, 1);
15567159e969SEd Cashin if (d == NULL) {
15577159e969SEd Cashin pr_info("aoe: device allocation failure\n");
15587159e969SEd Cashin return;
15597159e969SEd Cashin }
15607159e969SEd Cashin
15611da177e4SLinus Torvalds spin_lock_irqsave(&d->lock, flags);
15621da177e4SLinus Torvalds
156368e0d42fSEd L. Cashin t = gettgt(d, h->src);
15641b8a1636SEd Cashin if (t) {
15651b8a1636SEd Cashin t->nframes = n;
15661b8a1636SEd Cashin if (n < t->maxout)
15673a0c40d2SEd Cashin aoecmd_wreset(t);
15681b8a1636SEd Cashin } else {
156968e0d42fSEd L. Cashin t = addtgt(d, h->src, n);
157069cf2d85SEd Cashin if (!t)
157169cf2d85SEd Cashin goto bail;
157268e0d42fSEd L. Cashin }
15733f0f0133SEd Cashin n = skb->dev->mtu;
157419bf2635SEd L. Cashin n -= sizeof(struct aoe_hdr) + sizeof(struct aoe_atahdr);
157519bf2635SEd L. Cashin n /= 512;
157619bf2635SEd L. Cashin if (n > ch->scnt)
157719bf2635SEd L. Cashin n = ch->scnt;
15784f51dc5eSEd L. Cashin n = n ? n * 512 : DEFAULTBCNT;
15793f0f0133SEd Cashin setifbcnt(t, skb->dev, n);
15803ae1c24eSEd L. Cashin
15813ae1c24eSEd L. Cashin /* don't change users' perspective */
158269cf2d85SEd Cashin if (d->nopen == 0) {
158363e9cc5dSecashin@coraid.com d->fw_ver = be16_to_cpu(ch->fwver);
158468e0d42fSEd L. Cashin sl = aoecmd_ata_id(d);
158569cf2d85SEd Cashin }
158669cf2d85SEd Cashin bail:
15871da177e4SLinus Torvalds spin_unlock_irqrestore(&d->lock, flags);
158869cf2d85SEd Cashin aoedev_put(d);
1589e9bb8fb0SDavid S. Miller if (sl) {
1590e9bb8fb0SDavid S. Miller __skb_queue_head_init(&queue);
1591e9bb8fb0SDavid S. Miller __skb_queue_tail(&queue, sl);
1592e9bb8fb0SDavid S. Miller aoenet_xmit(&queue);
1593e9bb8fb0SDavid S. Miller }
15941da177e4SLinus Torvalds }
15951da177e4SLinus Torvalds
159668e0d42fSEd L. Cashin void
aoecmd_wreset(struct aoetgt * t)15973a0c40d2SEd Cashin aoecmd_wreset(struct aoetgt *t)
15983a0c40d2SEd Cashin {
15993a0c40d2SEd Cashin t->maxout = 1;
16003a0c40d2SEd Cashin t->ssthresh = t->nframes / 2;
16013a0c40d2SEd Cashin t->next_cwnd = t->nframes;
16023a0c40d2SEd Cashin }
16033a0c40d2SEd Cashin
16043a0c40d2SEd Cashin void
aoecmd_cleanslate(struct aoedev * d)160568e0d42fSEd L. Cashin aoecmd_cleanslate(struct aoedev *d)
160668e0d42fSEd L. Cashin {
160768e0d42fSEd L. Cashin struct aoetgt **t, **te;
160868e0d42fSEd L. Cashin
16093a0c40d2SEd Cashin d->rttavg = RTTAVG_INIT;
16103a0c40d2SEd Cashin d->rttdev = RTTDEV_INIT;
16113f0f0133SEd Cashin d->maxbcnt = 0;
161268e0d42fSEd L. Cashin
161368e0d42fSEd L. Cashin t = d->targets;
161471114ec4SEd Cashin te = t + d->ntargets;
16153f0f0133SEd Cashin for (; t < te && *t; t++)
16163a0c40d2SEd Cashin aoecmd_wreset(*t);
161768e0d42fSEd L. Cashin }
1618896831f5SEd Cashin
161969cf2d85SEd Cashin void
aoe_failbuf(struct aoedev * d,struct buf * buf)162069cf2d85SEd Cashin aoe_failbuf(struct aoedev *d, struct buf *buf)
162169cf2d85SEd Cashin {
162269cf2d85SEd Cashin if (buf == NULL)
162369cf2d85SEd Cashin return;
1624feb261e2SKent Overstreet buf->iter.bi_size = 0;
16254e4cbee9SChristoph Hellwig buf->bio->bi_status = BLK_STS_IOERR;
162669cf2d85SEd Cashin if (buf->nframesout == 0)
162769cf2d85SEd Cashin aoe_end_buf(d, buf);
162869cf2d85SEd Cashin }
162969cf2d85SEd Cashin
163069cf2d85SEd Cashin void
aoe_flush_iocq(void)163169cf2d85SEd Cashin aoe_flush_iocq(void)
1632896831f5SEd Cashin {
16338030d343SEd Cashin int i;
16348030d343SEd Cashin
16358030d343SEd Cashin for (i = 0; i < ncpus; i++) {
16368030d343SEd Cashin if (kts[i].active)
16378030d343SEd Cashin aoe_flush_iocq_by_index(i);
16388030d343SEd Cashin }
16398030d343SEd Cashin }
16408030d343SEd Cashin
16418030d343SEd Cashin void
aoe_flush_iocq_by_index(int id)16428030d343SEd Cashin aoe_flush_iocq_by_index(int id)
16438030d343SEd Cashin {
1644896831f5SEd Cashin struct frame *f;
1645896831f5SEd Cashin struct aoedev *d;
1646896831f5SEd Cashin LIST_HEAD(flist);
1647896831f5SEd Cashin struct list_head *pos;
1648896831f5SEd Cashin struct sk_buff *skb;
1649896831f5SEd Cashin ulong flags;
1650896831f5SEd Cashin
16518030d343SEd Cashin spin_lock_irqsave(&iocq[id].lock, flags);
16528030d343SEd Cashin list_splice_init(&iocq[id].head, &flist);
16538030d343SEd Cashin spin_unlock_irqrestore(&iocq[id].lock, flags);
1654896831f5SEd Cashin while (!list_empty(&flist)) {
1655896831f5SEd Cashin pos = flist.next;
1656896831f5SEd Cashin list_del(pos);
1657896831f5SEd Cashin f = list_entry(pos, struct frame, head);
1658896831f5SEd Cashin d = f->t->d;
1659896831f5SEd Cashin skb = f->r_skb;
1660896831f5SEd Cashin spin_lock_irqsave(&d->lock, flags);
1661896831f5SEd Cashin if (f->buf) {
1662896831f5SEd Cashin f->buf->nframesout--;
1663896831f5SEd Cashin aoe_failbuf(d, f->buf);
1664896831f5SEd Cashin }
1665896831f5SEd Cashin aoe_freetframe(f);
1666896831f5SEd Cashin spin_unlock_irqrestore(&d->lock, flags);
1667896831f5SEd Cashin dev_kfree_skb(skb);
166869cf2d85SEd Cashin aoedev_put(d);
1669896831f5SEd Cashin }
1670896831f5SEd Cashin }
1671896831f5SEd Cashin
1672896831f5SEd Cashin int __init
aoecmd_init(void)1673896831f5SEd Cashin aoecmd_init(void)
1674896831f5SEd Cashin {
1675bbb44e30SEd Cashin void *p;
16768030d343SEd Cashin int i;
16778030d343SEd Cashin int ret;
1678bbb44e30SEd Cashin
1679bbb44e30SEd Cashin /* get_zeroed_page returns page with ref count 1 */
168032d6bd90SMichal Hocko p = (void *) get_zeroed_page(GFP_KERNEL);
1681bbb44e30SEd Cashin if (!p)
1682bbb44e30SEd Cashin return -ENOMEM;
1683bbb44e30SEd Cashin empty_page = virt_to_page(p);
1684bbb44e30SEd Cashin
16858030d343SEd Cashin ncpus = num_online_cpus();
16868030d343SEd Cashin
16878030d343SEd Cashin iocq = kcalloc(ncpus, sizeof(struct iocq_ktio), GFP_KERNEL);
16888030d343SEd Cashin if (!iocq)
16898030d343SEd Cashin return -ENOMEM;
16908030d343SEd Cashin
16918030d343SEd Cashin kts = kcalloc(ncpus, sizeof(struct ktstate), GFP_KERNEL);
16928030d343SEd Cashin if (!kts) {
16938030d343SEd Cashin ret = -ENOMEM;
16948030d343SEd Cashin goto kts_fail;
16958030d343SEd Cashin }
16968030d343SEd Cashin
16978030d343SEd Cashin ktiowq = kcalloc(ncpus, sizeof(wait_queue_head_t), GFP_KERNEL);
16988030d343SEd Cashin if (!ktiowq) {
16998030d343SEd Cashin ret = -ENOMEM;
17008030d343SEd Cashin goto ktiowq_fail;
17018030d343SEd Cashin }
17028030d343SEd Cashin
17038030d343SEd Cashin for (i = 0; i < ncpus; i++) {
17048030d343SEd Cashin INIT_LIST_HEAD(&iocq[i].head);
17058030d343SEd Cashin spin_lock_init(&iocq[i].lock);
17068030d343SEd Cashin init_waitqueue_head(&ktiowq[i]);
17078030d343SEd Cashin snprintf(kts[i].name, sizeof(kts[i].name), "aoe_ktio%d", i);
17088030d343SEd Cashin kts[i].fn = ktio;
17098030d343SEd Cashin kts[i].waitq = &ktiowq[i];
17108030d343SEd Cashin kts[i].lock = &iocq[i].lock;
17118030d343SEd Cashin kts[i].id = i;
17128030d343SEd Cashin kts[i].active = 0;
17138030d343SEd Cashin }
17148030d343SEd Cashin kts[0].active = 1;
17158030d343SEd Cashin if (aoe_ktstart(&kts[0])) {
17168030d343SEd Cashin ret = -ENOMEM;
17178030d343SEd Cashin goto ktstart_fail;
17188030d343SEd Cashin }
17198030d343SEd Cashin return 0;
17208030d343SEd Cashin
17218030d343SEd Cashin ktstart_fail:
17228030d343SEd Cashin kfree(ktiowq);
17238030d343SEd Cashin ktiowq_fail:
17248030d343SEd Cashin kfree(kts);
17258030d343SEd Cashin kts_fail:
17268030d343SEd Cashin kfree(iocq);
17278030d343SEd Cashin
17288030d343SEd Cashin return ret;
1729896831f5SEd Cashin }
1730896831f5SEd Cashin
1731896831f5SEd Cashin void
aoecmd_exit(void)1732896831f5SEd Cashin aoecmd_exit(void)
1733896831f5SEd Cashin {
17348030d343SEd Cashin int i;
17358030d343SEd Cashin
17368030d343SEd Cashin for (i = 0; i < ncpus; i++)
17378030d343SEd Cashin if (kts[i].active)
17388030d343SEd Cashin aoe_ktstop(&kts[i]);
17398030d343SEd Cashin
174069cf2d85SEd Cashin aoe_flush_iocq();
1741bbb44e30SEd Cashin
17428030d343SEd Cashin /* Free up the iocq and thread speicific configuration
17438030d343SEd Cashin * allocated during startup.
17448030d343SEd Cashin */
17458030d343SEd Cashin kfree(iocq);
17468030d343SEd Cashin kfree(kts);
17478030d343SEd Cashin kfree(ktiowq);
17488030d343SEd Cashin
1749bbb44e30SEd Cashin free_page((unsigned long) page_address(empty_page));
1750bbb44e30SEd Cashin empty_page = NULL;
1751896831f5SEd Cashin }
1752