xref: /openbmc/linux/drivers/block/xen-blkfront.c (revision 4da722ca)
1 /*
2  * blkfront.c
3  *
4  * XenLinux virtual block device driver.
5  *
6  * Copyright (c) 2003-2004, Keir Fraser & Steve Hand
7  * Modifications by Mark A. Williamson are (c) Intel Research Cambridge
8  * Copyright (c) 2004, Christian Limpach
9  * Copyright (c) 2004, Andrew Warfield
10  * Copyright (c) 2005, Christopher Clark
11  * Copyright (c) 2005, XenSource Ltd
12  *
13  * This program is free software; you can redistribute it and/or
14  * modify it under the terms of the GNU General Public License version 2
15  * as published by the Free Software Foundation; or, when distributed
16  * separately from the Linux kernel or incorporated into other
17  * software packages, subject to the following license:
18  *
19  * Permission is hereby granted, free of charge, to any person obtaining a copy
20  * of this source file (the "Software"), to deal in the Software without
21  * restriction, including without limitation the rights to use, copy, modify,
22  * merge, publish, distribute, sublicense, and/or sell copies of the Software,
23  * and to permit persons to whom the Software is furnished to do so, subject to
24  * the following conditions:
25  *
26  * The above copyright notice and this permission notice shall be included in
27  * all copies or substantial portions of the Software.
28  *
29  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
30  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
31  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
32  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
33  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
34  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
35  * IN THE SOFTWARE.
36  */
37 
38 #include <linux/interrupt.h>
39 #include <linux/blkdev.h>
40 #include <linux/blk-mq.h>
41 #include <linux/hdreg.h>
42 #include <linux/cdrom.h>
43 #include <linux/module.h>
44 #include <linux/slab.h>
45 #include <linux/mutex.h>
46 #include <linux/scatterlist.h>
47 #include <linux/bitmap.h>
48 #include <linux/list.h>
49 
50 #include <xen/xen.h>
51 #include <xen/xenbus.h>
52 #include <xen/grant_table.h>
53 #include <xen/events.h>
54 #include <xen/page.h>
55 #include <xen/platform_pci.h>
56 
57 #include <xen/interface/grant_table.h>
58 #include <xen/interface/io/blkif.h>
59 #include <xen/interface/io/protocols.h>
60 
61 #include <asm/xen/hypervisor.h>
62 
63 /*
64  * The minimal size of segment supported by the block framework is PAGE_SIZE.
65  * When Linux is using a different page size than Xen, it may not be possible
66  * to put all the data in a single segment.
67  * This can happen when the backend doesn't support indirect descriptor and
68  * therefore the maximum amount of data that a request can carry is
69  * BLKIF_MAX_SEGMENTS_PER_REQUEST * XEN_PAGE_SIZE = 44KB
70  *
71  * Note that we only support one extra request. So the Linux page size
72  * should be <= ( 2 * BLKIF_MAX_SEGMENTS_PER_REQUEST * XEN_PAGE_SIZE) =
73  * 88KB.
74  */
75 #define HAS_EXTRA_REQ (BLKIF_MAX_SEGMENTS_PER_REQUEST < XEN_PFN_PER_PAGE)
76 
77 enum blkif_state {
78 	BLKIF_STATE_DISCONNECTED,
79 	BLKIF_STATE_CONNECTED,
80 	BLKIF_STATE_SUSPENDED,
81 };
82 
83 struct grant {
84 	grant_ref_t gref;
85 	struct page *page;
86 	struct list_head node;
87 };
88 
89 enum blk_req_status {
90 	REQ_WAITING,
91 	REQ_DONE,
92 	REQ_ERROR,
93 	REQ_EOPNOTSUPP,
94 };
95 
96 struct blk_shadow {
97 	struct blkif_request req;
98 	struct request *request;
99 	struct grant **grants_used;
100 	struct grant **indirect_grants;
101 	struct scatterlist *sg;
102 	unsigned int num_sg;
103 	enum blk_req_status status;
104 
105 	#define NO_ASSOCIATED_ID ~0UL
106 	/*
107 	 * Id of the sibling if we ever need 2 requests when handling a
108 	 * block I/O request
109 	 */
110 	unsigned long associated_id;
111 };
112 
113 struct blkif_req {
114 	int	error;
115 };
116 
117 static inline struct blkif_req *blkif_req(struct request *rq)
118 {
119 	return blk_mq_rq_to_pdu(rq);
120 }
121 
122 static DEFINE_MUTEX(blkfront_mutex);
123 static const struct block_device_operations xlvbd_block_fops;
124 
125 /*
126  * Maximum number of segments in indirect requests, the actual value used by
127  * the frontend driver is the minimum of this value and the value provided
128  * by the backend driver.
129  */
130 
131 static unsigned int xen_blkif_max_segments = 32;
132 module_param_named(max_indirect_segments, xen_blkif_max_segments, uint,
133 		   S_IRUGO);
134 MODULE_PARM_DESC(max_indirect_segments,
135 		 "Maximum amount of segments in indirect requests (default is 32)");
136 
137 static unsigned int xen_blkif_max_queues = 4;
138 module_param_named(max_queues, xen_blkif_max_queues, uint, S_IRUGO);
139 MODULE_PARM_DESC(max_queues, "Maximum number of hardware queues/rings used per virtual disk");
140 
141 /*
142  * Maximum order of pages to be used for the shared ring between front and
143  * backend, 4KB page granularity is used.
144  */
145 static unsigned int xen_blkif_max_ring_order;
146 module_param_named(max_ring_page_order, xen_blkif_max_ring_order, int, S_IRUGO);
147 MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the shared ring");
148 
149 #define BLK_RING_SIZE(info)	\
150 	__CONST_RING_SIZE(blkif, XEN_PAGE_SIZE * (info)->nr_ring_pages)
151 
152 #define BLK_MAX_RING_SIZE	\
153 	__CONST_RING_SIZE(blkif, XEN_PAGE_SIZE * XENBUS_MAX_RING_GRANTS)
154 
155 /*
156  * ring-ref%u i=(-1UL) would take 11 characters + 'ring-ref' is 8, so 19
157  * characters are enough. Define to 20 to keep consistent with backend.
158  */
159 #define RINGREF_NAME_LEN (20)
160 /*
161  * queue-%u would take 7 + 10(UINT_MAX) = 17 characters.
162  */
163 #define QUEUE_NAME_LEN (17)
164 
165 /*
166  *  Per-ring info.
167  *  Every blkfront device can associate with one or more blkfront_ring_info,
168  *  depending on how many hardware queues/rings to be used.
169  */
170 struct blkfront_ring_info {
171 	/* Lock to protect data in every ring buffer. */
172 	spinlock_t ring_lock;
173 	struct blkif_front_ring ring;
174 	unsigned int ring_ref[XENBUS_MAX_RING_GRANTS];
175 	unsigned int evtchn, irq;
176 	struct work_struct work;
177 	struct gnttab_free_callback callback;
178 	struct blk_shadow shadow[BLK_MAX_RING_SIZE];
179 	struct list_head indirect_pages;
180 	struct list_head grants;
181 	unsigned int persistent_gnts_c;
182 	unsigned long shadow_free;
183 	struct blkfront_info *dev_info;
184 };
185 
186 /*
187  * We have one of these per vbd, whether ide, scsi or 'other'.  They
188  * hang in private_data off the gendisk structure. We may end up
189  * putting all kinds of interesting stuff here :-)
190  */
191 struct blkfront_info
192 {
193 	struct mutex mutex;
194 	struct xenbus_device *xbdev;
195 	struct gendisk *gd;
196 	u16 sector_size;
197 	unsigned int physical_sector_size;
198 	int vdevice;
199 	blkif_vdev_t handle;
200 	enum blkif_state connected;
201 	/* Number of pages per ring buffer. */
202 	unsigned int nr_ring_pages;
203 	struct request_queue *rq;
204 	unsigned int feature_flush:1;
205 	unsigned int feature_fua:1;
206 	unsigned int feature_discard:1;
207 	unsigned int feature_secdiscard:1;
208 	unsigned int feature_persistent:1;
209 	unsigned int discard_granularity;
210 	unsigned int discard_alignment;
211 	/* Number of 4KB segments handled */
212 	unsigned int max_indirect_segments;
213 	int is_ready;
214 	struct blk_mq_tag_set tag_set;
215 	struct blkfront_ring_info *rinfo;
216 	unsigned int nr_rings;
217 	/* Save uncomplete reqs and bios for migration. */
218 	struct list_head requests;
219 	struct bio_list bio_list;
220 };
221 
222 static unsigned int nr_minors;
223 static unsigned long *minors;
224 static DEFINE_SPINLOCK(minor_lock);
225 
226 #define GRANT_INVALID_REF	0
227 
228 #define PARTS_PER_DISK		16
229 #define PARTS_PER_EXT_DISK      256
230 
231 #define BLKIF_MAJOR(dev) ((dev)>>8)
232 #define BLKIF_MINOR(dev) ((dev) & 0xff)
233 
234 #define EXT_SHIFT 28
235 #define EXTENDED (1<<EXT_SHIFT)
236 #define VDEV_IS_EXTENDED(dev) ((dev)&(EXTENDED))
237 #define BLKIF_MINOR_EXT(dev) ((dev)&(~EXTENDED))
238 #define EMULATED_HD_DISK_MINOR_OFFSET (0)
239 #define EMULATED_HD_DISK_NAME_OFFSET (EMULATED_HD_DISK_MINOR_OFFSET / 256)
240 #define EMULATED_SD_DISK_MINOR_OFFSET (0)
241 #define EMULATED_SD_DISK_NAME_OFFSET (EMULATED_SD_DISK_MINOR_OFFSET / 256)
242 
243 #define DEV_NAME	"xvd"	/* name in /dev */
244 
245 /*
246  * Grants are always the same size as a Xen page (i.e 4KB).
247  * A physical segment is always the same size as a Linux page.
248  * Number of grants per physical segment
249  */
250 #define GRANTS_PER_PSEG	(PAGE_SIZE / XEN_PAGE_SIZE)
251 
252 #define GRANTS_PER_INDIRECT_FRAME \
253 	(XEN_PAGE_SIZE / sizeof(struct blkif_request_segment))
254 
255 #define PSEGS_PER_INDIRECT_FRAME	\
256 	(GRANTS_INDIRECT_FRAME / GRANTS_PSEGS)
257 
258 #define INDIRECT_GREFS(_grants)		\
259 	DIV_ROUND_UP(_grants, GRANTS_PER_INDIRECT_FRAME)
260 
261 #define GREFS(_psegs)	((_psegs) * GRANTS_PER_PSEG)
262 
263 static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo);
264 static void blkfront_gather_backend_features(struct blkfront_info *info);
265 
266 static int get_id_from_freelist(struct blkfront_ring_info *rinfo)
267 {
268 	unsigned long free = rinfo->shadow_free;
269 
270 	BUG_ON(free >= BLK_RING_SIZE(rinfo->dev_info));
271 	rinfo->shadow_free = rinfo->shadow[free].req.u.rw.id;
272 	rinfo->shadow[free].req.u.rw.id = 0x0fffffee; /* debug */
273 	return free;
274 }
275 
276 static int add_id_to_freelist(struct blkfront_ring_info *rinfo,
277 			      unsigned long id)
278 {
279 	if (rinfo->shadow[id].req.u.rw.id != id)
280 		return -EINVAL;
281 	if (rinfo->shadow[id].request == NULL)
282 		return -EINVAL;
283 	rinfo->shadow[id].req.u.rw.id  = rinfo->shadow_free;
284 	rinfo->shadow[id].request = NULL;
285 	rinfo->shadow_free = id;
286 	return 0;
287 }
288 
289 static int fill_grant_buffer(struct blkfront_ring_info *rinfo, int num)
290 {
291 	struct blkfront_info *info = rinfo->dev_info;
292 	struct page *granted_page;
293 	struct grant *gnt_list_entry, *n;
294 	int i = 0;
295 
296 	while (i < num) {
297 		gnt_list_entry = kzalloc(sizeof(struct grant), GFP_NOIO);
298 		if (!gnt_list_entry)
299 			goto out_of_memory;
300 
301 		if (info->feature_persistent) {
302 			granted_page = alloc_page(GFP_NOIO);
303 			if (!granted_page) {
304 				kfree(gnt_list_entry);
305 				goto out_of_memory;
306 			}
307 			gnt_list_entry->page = granted_page;
308 		}
309 
310 		gnt_list_entry->gref = GRANT_INVALID_REF;
311 		list_add(&gnt_list_entry->node, &rinfo->grants);
312 		i++;
313 	}
314 
315 	return 0;
316 
317 out_of_memory:
318 	list_for_each_entry_safe(gnt_list_entry, n,
319 	                         &rinfo->grants, node) {
320 		list_del(&gnt_list_entry->node);
321 		if (info->feature_persistent)
322 			__free_page(gnt_list_entry->page);
323 		kfree(gnt_list_entry);
324 		i--;
325 	}
326 	BUG_ON(i != 0);
327 	return -ENOMEM;
328 }
329 
330 static struct grant *get_free_grant(struct blkfront_ring_info *rinfo)
331 {
332 	struct grant *gnt_list_entry;
333 
334 	BUG_ON(list_empty(&rinfo->grants));
335 	gnt_list_entry = list_first_entry(&rinfo->grants, struct grant,
336 					  node);
337 	list_del(&gnt_list_entry->node);
338 
339 	if (gnt_list_entry->gref != GRANT_INVALID_REF)
340 		rinfo->persistent_gnts_c--;
341 
342 	return gnt_list_entry;
343 }
344 
345 static inline void grant_foreign_access(const struct grant *gnt_list_entry,
346 					const struct blkfront_info *info)
347 {
348 	gnttab_page_grant_foreign_access_ref_one(gnt_list_entry->gref,
349 						 info->xbdev->otherend_id,
350 						 gnt_list_entry->page,
351 						 0);
352 }
353 
354 static struct grant *get_grant(grant_ref_t *gref_head,
355 			       unsigned long gfn,
356 			       struct blkfront_ring_info *rinfo)
357 {
358 	struct grant *gnt_list_entry = get_free_grant(rinfo);
359 	struct blkfront_info *info = rinfo->dev_info;
360 
361 	if (gnt_list_entry->gref != GRANT_INVALID_REF)
362 		return gnt_list_entry;
363 
364 	/* Assign a gref to this page */
365 	gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head);
366 	BUG_ON(gnt_list_entry->gref == -ENOSPC);
367 	if (info->feature_persistent)
368 		grant_foreign_access(gnt_list_entry, info);
369 	else {
370 		/* Grant access to the GFN passed by the caller */
371 		gnttab_grant_foreign_access_ref(gnt_list_entry->gref,
372 						info->xbdev->otherend_id,
373 						gfn, 0);
374 	}
375 
376 	return gnt_list_entry;
377 }
378 
379 static struct grant *get_indirect_grant(grant_ref_t *gref_head,
380 					struct blkfront_ring_info *rinfo)
381 {
382 	struct grant *gnt_list_entry = get_free_grant(rinfo);
383 	struct blkfront_info *info = rinfo->dev_info;
384 
385 	if (gnt_list_entry->gref != GRANT_INVALID_REF)
386 		return gnt_list_entry;
387 
388 	/* Assign a gref to this page */
389 	gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head);
390 	BUG_ON(gnt_list_entry->gref == -ENOSPC);
391 	if (!info->feature_persistent) {
392 		struct page *indirect_page;
393 
394 		/* Fetch a pre-allocated page to use for indirect grefs */
395 		BUG_ON(list_empty(&rinfo->indirect_pages));
396 		indirect_page = list_first_entry(&rinfo->indirect_pages,
397 						 struct page, lru);
398 		list_del(&indirect_page->lru);
399 		gnt_list_entry->page = indirect_page;
400 	}
401 	grant_foreign_access(gnt_list_entry, info);
402 
403 	return gnt_list_entry;
404 }
405 
406 static const char *op_name(int op)
407 {
408 	static const char *const names[] = {
409 		[BLKIF_OP_READ] = "read",
410 		[BLKIF_OP_WRITE] = "write",
411 		[BLKIF_OP_WRITE_BARRIER] = "barrier",
412 		[BLKIF_OP_FLUSH_DISKCACHE] = "flush",
413 		[BLKIF_OP_DISCARD] = "discard" };
414 
415 	if (op < 0 || op >= ARRAY_SIZE(names))
416 		return "unknown";
417 
418 	if (!names[op])
419 		return "reserved";
420 
421 	return names[op];
422 }
423 static int xlbd_reserve_minors(unsigned int minor, unsigned int nr)
424 {
425 	unsigned int end = minor + nr;
426 	int rc;
427 
428 	if (end > nr_minors) {
429 		unsigned long *bitmap, *old;
430 
431 		bitmap = kcalloc(BITS_TO_LONGS(end), sizeof(*bitmap),
432 				 GFP_KERNEL);
433 		if (bitmap == NULL)
434 			return -ENOMEM;
435 
436 		spin_lock(&minor_lock);
437 		if (end > nr_minors) {
438 			old = minors;
439 			memcpy(bitmap, minors,
440 			       BITS_TO_LONGS(nr_minors) * sizeof(*bitmap));
441 			minors = bitmap;
442 			nr_minors = BITS_TO_LONGS(end) * BITS_PER_LONG;
443 		} else
444 			old = bitmap;
445 		spin_unlock(&minor_lock);
446 		kfree(old);
447 	}
448 
449 	spin_lock(&minor_lock);
450 	if (find_next_bit(minors, end, minor) >= end) {
451 		bitmap_set(minors, minor, nr);
452 		rc = 0;
453 	} else
454 		rc = -EBUSY;
455 	spin_unlock(&minor_lock);
456 
457 	return rc;
458 }
459 
460 static void xlbd_release_minors(unsigned int minor, unsigned int nr)
461 {
462 	unsigned int end = minor + nr;
463 
464 	BUG_ON(end > nr_minors);
465 	spin_lock(&minor_lock);
466 	bitmap_clear(minors,  minor, nr);
467 	spin_unlock(&minor_lock);
468 }
469 
470 static void blkif_restart_queue_callback(void *arg)
471 {
472 	struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)arg;
473 	schedule_work(&rinfo->work);
474 }
475 
476 static int blkif_getgeo(struct block_device *bd, struct hd_geometry *hg)
477 {
478 	/* We don't have real geometry info, but let's at least return
479 	   values consistent with the size of the device */
480 	sector_t nsect = get_capacity(bd->bd_disk);
481 	sector_t cylinders = nsect;
482 
483 	hg->heads = 0xff;
484 	hg->sectors = 0x3f;
485 	sector_div(cylinders, hg->heads * hg->sectors);
486 	hg->cylinders = cylinders;
487 	if ((sector_t)(hg->cylinders + 1) * hg->heads * hg->sectors < nsect)
488 		hg->cylinders = 0xffff;
489 	return 0;
490 }
491 
492 static int blkif_ioctl(struct block_device *bdev, fmode_t mode,
493 		       unsigned command, unsigned long argument)
494 {
495 	struct blkfront_info *info = bdev->bd_disk->private_data;
496 	int i;
497 
498 	dev_dbg(&info->xbdev->dev, "command: 0x%x, argument: 0x%lx\n",
499 		command, (long)argument);
500 
501 	switch (command) {
502 	case CDROMMULTISESSION:
503 		dev_dbg(&info->xbdev->dev, "FIXME: support multisession CDs later\n");
504 		for (i = 0; i < sizeof(struct cdrom_multisession); i++)
505 			if (put_user(0, (char __user *)(argument + i)))
506 				return -EFAULT;
507 		return 0;
508 
509 	case CDROM_GET_CAPABILITY: {
510 		struct gendisk *gd = info->gd;
511 		if (gd->flags & GENHD_FL_CD)
512 			return 0;
513 		return -EINVAL;
514 	}
515 
516 	default:
517 		/*printk(KERN_ALERT "ioctl %08x not supported by Xen blkdev\n",
518 		  command);*/
519 		return -EINVAL; /* same return as native Linux */
520 	}
521 
522 	return 0;
523 }
524 
525 static unsigned long blkif_ring_get_request(struct blkfront_ring_info *rinfo,
526 					    struct request *req,
527 					    struct blkif_request **ring_req)
528 {
529 	unsigned long id;
530 
531 	*ring_req = RING_GET_REQUEST(&rinfo->ring, rinfo->ring.req_prod_pvt);
532 	rinfo->ring.req_prod_pvt++;
533 
534 	id = get_id_from_freelist(rinfo);
535 	rinfo->shadow[id].request = req;
536 	rinfo->shadow[id].status = REQ_WAITING;
537 	rinfo->shadow[id].associated_id = NO_ASSOCIATED_ID;
538 
539 	(*ring_req)->u.rw.id = id;
540 
541 	return id;
542 }
543 
544 static int blkif_queue_discard_req(struct request *req, struct blkfront_ring_info *rinfo)
545 {
546 	struct blkfront_info *info = rinfo->dev_info;
547 	struct blkif_request *ring_req;
548 	unsigned long id;
549 
550 	/* Fill out a communications ring structure. */
551 	id = blkif_ring_get_request(rinfo, req, &ring_req);
552 
553 	ring_req->operation = BLKIF_OP_DISCARD;
554 	ring_req->u.discard.nr_sectors = blk_rq_sectors(req);
555 	ring_req->u.discard.id = id;
556 	ring_req->u.discard.sector_number = (blkif_sector_t)blk_rq_pos(req);
557 	if (req_op(req) == REQ_OP_SECURE_ERASE && info->feature_secdiscard)
558 		ring_req->u.discard.flag = BLKIF_DISCARD_SECURE;
559 	else
560 		ring_req->u.discard.flag = 0;
561 
562 	/* Keep a private copy so we can reissue requests when recovering. */
563 	rinfo->shadow[id].req = *ring_req;
564 
565 	return 0;
566 }
567 
568 struct setup_rw_req {
569 	unsigned int grant_idx;
570 	struct blkif_request_segment *segments;
571 	struct blkfront_ring_info *rinfo;
572 	struct blkif_request *ring_req;
573 	grant_ref_t gref_head;
574 	unsigned int id;
575 	/* Only used when persistent grant is used and it's a read request */
576 	bool need_copy;
577 	unsigned int bvec_off;
578 	char *bvec_data;
579 
580 	bool require_extra_req;
581 	struct blkif_request *extra_ring_req;
582 };
583 
584 static void blkif_setup_rw_req_grant(unsigned long gfn, unsigned int offset,
585 				     unsigned int len, void *data)
586 {
587 	struct setup_rw_req *setup = data;
588 	int n, ref;
589 	struct grant *gnt_list_entry;
590 	unsigned int fsect, lsect;
591 	/* Convenient aliases */
592 	unsigned int grant_idx = setup->grant_idx;
593 	struct blkif_request *ring_req = setup->ring_req;
594 	struct blkfront_ring_info *rinfo = setup->rinfo;
595 	/*
596 	 * We always use the shadow of the first request to store the list
597 	 * of grant associated to the block I/O request. This made the
598 	 * completion more easy to handle even if the block I/O request is
599 	 * split.
600 	 */
601 	struct blk_shadow *shadow = &rinfo->shadow[setup->id];
602 
603 	if (unlikely(setup->require_extra_req &&
604 		     grant_idx >= BLKIF_MAX_SEGMENTS_PER_REQUEST)) {
605 		/*
606 		 * We are using the second request, setup grant_idx
607 		 * to be the index of the segment array.
608 		 */
609 		grant_idx -= BLKIF_MAX_SEGMENTS_PER_REQUEST;
610 		ring_req = setup->extra_ring_req;
611 	}
612 
613 	if ((ring_req->operation == BLKIF_OP_INDIRECT) &&
614 	    (grant_idx % GRANTS_PER_INDIRECT_FRAME == 0)) {
615 		if (setup->segments)
616 			kunmap_atomic(setup->segments);
617 
618 		n = grant_idx / GRANTS_PER_INDIRECT_FRAME;
619 		gnt_list_entry = get_indirect_grant(&setup->gref_head, rinfo);
620 		shadow->indirect_grants[n] = gnt_list_entry;
621 		setup->segments = kmap_atomic(gnt_list_entry->page);
622 		ring_req->u.indirect.indirect_grefs[n] = gnt_list_entry->gref;
623 	}
624 
625 	gnt_list_entry = get_grant(&setup->gref_head, gfn, rinfo);
626 	ref = gnt_list_entry->gref;
627 	/*
628 	 * All the grants are stored in the shadow of the first
629 	 * request. Therefore we have to use the global index.
630 	 */
631 	shadow->grants_used[setup->grant_idx] = gnt_list_entry;
632 
633 	if (setup->need_copy) {
634 		void *shared_data;
635 
636 		shared_data = kmap_atomic(gnt_list_entry->page);
637 		/*
638 		 * this does not wipe data stored outside the
639 		 * range sg->offset..sg->offset+sg->length.
640 		 * Therefore, blkback *could* see data from
641 		 * previous requests. This is OK as long as
642 		 * persistent grants are shared with just one
643 		 * domain. It may need refactoring if this
644 		 * changes
645 		 */
646 		memcpy(shared_data + offset,
647 		       setup->bvec_data + setup->bvec_off,
648 		       len);
649 
650 		kunmap_atomic(shared_data);
651 		setup->bvec_off += len;
652 	}
653 
654 	fsect = offset >> 9;
655 	lsect = fsect + (len >> 9) - 1;
656 	if (ring_req->operation != BLKIF_OP_INDIRECT) {
657 		ring_req->u.rw.seg[grant_idx] =
658 			(struct blkif_request_segment) {
659 				.gref       = ref,
660 				.first_sect = fsect,
661 				.last_sect  = lsect };
662 	} else {
663 		setup->segments[grant_idx % GRANTS_PER_INDIRECT_FRAME] =
664 			(struct blkif_request_segment) {
665 				.gref       = ref,
666 				.first_sect = fsect,
667 				.last_sect  = lsect };
668 	}
669 
670 	(setup->grant_idx)++;
671 }
672 
673 static void blkif_setup_extra_req(struct blkif_request *first,
674 				  struct blkif_request *second)
675 {
676 	uint16_t nr_segments = first->u.rw.nr_segments;
677 
678 	/*
679 	 * The second request is only present when the first request uses
680 	 * all its segments. It's always the continuity of the first one.
681 	 */
682 	first->u.rw.nr_segments = BLKIF_MAX_SEGMENTS_PER_REQUEST;
683 
684 	second->u.rw.nr_segments = nr_segments - BLKIF_MAX_SEGMENTS_PER_REQUEST;
685 	second->u.rw.sector_number = first->u.rw.sector_number +
686 		(BLKIF_MAX_SEGMENTS_PER_REQUEST * XEN_PAGE_SIZE) / 512;
687 
688 	second->u.rw.handle = first->u.rw.handle;
689 	second->operation = first->operation;
690 }
691 
692 static int blkif_queue_rw_req(struct request *req, struct blkfront_ring_info *rinfo)
693 {
694 	struct blkfront_info *info = rinfo->dev_info;
695 	struct blkif_request *ring_req, *extra_ring_req = NULL;
696 	unsigned long id, extra_id = NO_ASSOCIATED_ID;
697 	bool require_extra_req = false;
698 	int i;
699 	struct setup_rw_req setup = {
700 		.grant_idx = 0,
701 		.segments = NULL,
702 		.rinfo = rinfo,
703 		.need_copy = rq_data_dir(req) && info->feature_persistent,
704 	};
705 
706 	/*
707 	 * Used to store if we are able to queue the request by just using
708 	 * existing persistent grants, or if we have to get new grants,
709 	 * as there are not sufficiently many free.
710 	 */
711 	struct scatterlist *sg;
712 	int num_sg, max_grefs, num_grant;
713 
714 	max_grefs = req->nr_phys_segments * GRANTS_PER_PSEG;
715 	if (max_grefs > BLKIF_MAX_SEGMENTS_PER_REQUEST)
716 		/*
717 		 * If we are using indirect segments we need to account
718 		 * for the indirect grefs used in the request.
719 		 */
720 		max_grefs += INDIRECT_GREFS(max_grefs);
721 
722 	/*
723 	 * We have to reserve 'max_grefs' grants because persistent
724 	 * grants are shared by all rings.
725 	 */
726 	if (max_grefs > 0)
727 		if (gnttab_alloc_grant_references(max_grefs, &setup.gref_head) < 0) {
728 			gnttab_request_free_callback(
729 				&rinfo->callback,
730 				blkif_restart_queue_callback,
731 				rinfo,
732 				max_grefs);
733 			return 1;
734 		}
735 
736 	/* Fill out a communications ring structure. */
737 	id = blkif_ring_get_request(rinfo, req, &ring_req);
738 
739 	num_sg = blk_rq_map_sg(req->q, req, rinfo->shadow[id].sg);
740 	num_grant = 0;
741 	/* Calculate the number of grant used */
742 	for_each_sg(rinfo->shadow[id].sg, sg, num_sg, i)
743 	       num_grant += gnttab_count_grant(sg->offset, sg->length);
744 
745 	require_extra_req = info->max_indirect_segments == 0 &&
746 		num_grant > BLKIF_MAX_SEGMENTS_PER_REQUEST;
747 	BUG_ON(!HAS_EXTRA_REQ && require_extra_req);
748 
749 	rinfo->shadow[id].num_sg = num_sg;
750 	if (num_grant > BLKIF_MAX_SEGMENTS_PER_REQUEST &&
751 	    likely(!require_extra_req)) {
752 		/*
753 		 * The indirect operation can only be a BLKIF_OP_READ or
754 		 * BLKIF_OP_WRITE
755 		 */
756 		BUG_ON(req_op(req) == REQ_OP_FLUSH || req->cmd_flags & REQ_FUA);
757 		ring_req->operation = BLKIF_OP_INDIRECT;
758 		ring_req->u.indirect.indirect_op = rq_data_dir(req) ?
759 			BLKIF_OP_WRITE : BLKIF_OP_READ;
760 		ring_req->u.indirect.sector_number = (blkif_sector_t)blk_rq_pos(req);
761 		ring_req->u.indirect.handle = info->handle;
762 		ring_req->u.indirect.nr_segments = num_grant;
763 	} else {
764 		ring_req->u.rw.sector_number = (blkif_sector_t)blk_rq_pos(req);
765 		ring_req->u.rw.handle = info->handle;
766 		ring_req->operation = rq_data_dir(req) ?
767 			BLKIF_OP_WRITE : BLKIF_OP_READ;
768 		if (req_op(req) == REQ_OP_FLUSH || req->cmd_flags & REQ_FUA) {
769 			/*
770 			 * Ideally we can do an unordered flush-to-disk.
771 			 * In case the backend onlysupports barriers, use that.
772 			 * A barrier request a superset of FUA, so we can
773 			 * implement it the same way.  (It's also a FLUSH+FUA,
774 			 * since it is guaranteed ordered WRT previous writes.)
775 			 */
776 			if (info->feature_flush && info->feature_fua)
777 				ring_req->operation =
778 					BLKIF_OP_WRITE_BARRIER;
779 			else if (info->feature_flush)
780 				ring_req->operation =
781 					BLKIF_OP_FLUSH_DISKCACHE;
782 			else
783 				ring_req->operation = 0;
784 		}
785 		ring_req->u.rw.nr_segments = num_grant;
786 		if (unlikely(require_extra_req)) {
787 			extra_id = blkif_ring_get_request(rinfo, req,
788 							  &extra_ring_req);
789 			/*
790 			 * Only the first request contains the scatter-gather
791 			 * list.
792 			 */
793 			rinfo->shadow[extra_id].num_sg = 0;
794 
795 			blkif_setup_extra_req(ring_req, extra_ring_req);
796 
797 			/* Link the 2 requests together */
798 			rinfo->shadow[extra_id].associated_id = id;
799 			rinfo->shadow[id].associated_id = extra_id;
800 		}
801 	}
802 
803 	setup.ring_req = ring_req;
804 	setup.id = id;
805 
806 	setup.require_extra_req = require_extra_req;
807 	if (unlikely(require_extra_req))
808 		setup.extra_ring_req = extra_ring_req;
809 
810 	for_each_sg(rinfo->shadow[id].sg, sg, num_sg, i) {
811 		BUG_ON(sg->offset + sg->length > PAGE_SIZE);
812 
813 		if (setup.need_copy) {
814 			setup.bvec_off = sg->offset;
815 			setup.bvec_data = kmap_atomic(sg_page(sg));
816 		}
817 
818 		gnttab_foreach_grant_in_range(sg_page(sg),
819 					      sg->offset,
820 					      sg->length,
821 					      blkif_setup_rw_req_grant,
822 					      &setup);
823 
824 		if (setup.need_copy)
825 			kunmap_atomic(setup.bvec_data);
826 	}
827 	if (setup.segments)
828 		kunmap_atomic(setup.segments);
829 
830 	/* Keep a private copy so we can reissue requests when recovering. */
831 	rinfo->shadow[id].req = *ring_req;
832 	if (unlikely(require_extra_req))
833 		rinfo->shadow[extra_id].req = *extra_ring_req;
834 
835 	if (max_grefs > 0)
836 		gnttab_free_grant_references(setup.gref_head);
837 
838 	return 0;
839 }
840 
841 /*
842  * Generate a Xen blkfront IO request from a blk layer request.  Reads
843  * and writes are handled as expected.
844  *
845  * @req: a request struct
846  */
847 static int blkif_queue_request(struct request *req, struct blkfront_ring_info *rinfo)
848 {
849 	if (unlikely(rinfo->dev_info->connected != BLKIF_STATE_CONNECTED))
850 		return 1;
851 
852 	if (unlikely(req_op(req) == REQ_OP_DISCARD ||
853 		     req_op(req) == REQ_OP_SECURE_ERASE))
854 		return blkif_queue_discard_req(req, rinfo);
855 	else
856 		return blkif_queue_rw_req(req, rinfo);
857 }
858 
859 static inline void flush_requests(struct blkfront_ring_info *rinfo)
860 {
861 	int notify;
862 
863 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&rinfo->ring, notify);
864 
865 	if (notify)
866 		notify_remote_via_irq(rinfo->irq);
867 }
868 
869 static inline bool blkif_request_flush_invalid(struct request *req,
870 					       struct blkfront_info *info)
871 {
872 	return (blk_rq_is_passthrough(req) ||
873 		((req_op(req) == REQ_OP_FLUSH) &&
874 		 !info->feature_flush) ||
875 		((req->cmd_flags & REQ_FUA) &&
876 		 !info->feature_fua));
877 }
878 
879 static blk_status_t blkif_queue_rq(struct blk_mq_hw_ctx *hctx,
880 			  const struct blk_mq_queue_data *qd)
881 {
882 	unsigned long flags;
883 	int qid = hctx->queue_num;
884 	struct blkfront_info *info = hctx->queue->queuedata;
885 	struct blkfront_ring_info *rinfo = NULL;
886 
887 	BUG_ON(info->nr_rings <= qid);
888 	rinfo = &info->rinfo[qid];
889 	blk_mq_start_request(qd->rq);
890 	spin_lock_irqsave(&rinfo->ring_lock, flags);
891 	if (RING_FULL(&rinfo->ring))
892 		goto out_busy;
893 
894 	if (blkif_request_flush_invalid(qd->rq, rinfo->dev_info))
895 		goto out_err;
896 
897 	if (blkif_queue_request(qd->rq, rinfo))
898 		goto out_busy;
899 
900 	flush_requests(rinfo);
901 	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
902 	return BLK_STS_OK;
903 
904 out_err:
905 	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
906 	return BLK_STS_IOERR;
907 
908 out_busy:
909 	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
910 	blk_mq_stop_hw_queue(hctx);
911 	return BLK_STS_RESOURCE;
912 }
913 
914 static void blkif_complete_rq(struct request *rq)
915 {
916 	blk_mq_end_request(rq, blkif_req(rq)->error);
917 }
918 
919 static const struct blk_mq_ops blkfront_mq_ops = {
920 	.queue_rq = blkif_queue_rq,
921 	.complete = blkif_complete_rq,
922 };
923 
924 static void blkif_set_queue_limits(struct blkfront_info *info)
925 {
926 	struct request_queue *rq = info->rq;
927 	struct gendisk *gd = info->gd;
928 	unsigned int segments = info->max_indirect_segments ? :
929 				BLKIF_MAX_SEGMENTS_PER_REQUEST;
930 
931 	queue_flag_set_unlocked(QUEUE_FLAG_VIRT, rq);
932 
933 	if (info->feature_discard) {
934 		queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, rq);
935 		blk_queue_max_discard_sectors(rq, get_capacity(gd));
936 		rq->limits.discard_granularity = info->discard_granularity;
937 		rq->limits.discard_alignment = info->discard_alignment;
938 		if (info->feature_secdiscard)
939 			queue_flag_set_unlocked(QUEUE_FLAG_SECERASE, rq);
940 	}
941 
942 	/* Hard sector size and max sectors impersonate the equiv. hardware. */
943 	blk_queue_logical_block_size(rq, info->sector_size);
944 	blk_queue_physical_block_size(rq, info->physical_sector_size);
945 	blk_queue_max_hw_sectors(rq, (segments * XEN_PAGE_SIZE) / 512);
946 
947 	/* Each segment in a request is up to an aligned page in size. */
948 	blk_queue_segment_boundary(rq, PAGE_SIZE - 1);
949 	blk_queue_max_segment_size(rq, PAGE_SIZE);
950 
951 	/* Ensure a merged request will fit in a single I/O ring slot. */
952 	blk_queue_max_segments(rq, segments / GRANTS_PER_PSEG);
953 
954 	/* Make sure buffer addresses are sector-aligned. */
955 	blk_queue_dma_alignment(rq, 511);
956 }
957 
958 static int xlvbd_init_blk_queue(struct gendisk *gd, u16 sector_size,
959 				unsigned int physical_sector_size)
960 {
961 	struct request_queue *rq;
962 	struct blkfront_info *info = gd->private_data;
963 
964 	memset(&info->tag_set, 0, sizeof(info->tag_set));
965 	info->tag_set.ops = &blkfront_mq_ops;
966 	info->tag_set.nr_hw_queues = info->nr_rings;
967 	if (HAS_EXTRA_REQ && info->max_indirect_segments == 0) {
968 		/*
969 		 * When indirect descriptior is not supported, the I/O request
970 		 * will be split between multiple request in the ring.
971 		 * To avoid problems when sending the request, divide by
972 		 * 2 the depth of the queue.
973 		 */
974 		info->tag_set.queue_depth =  BLK_RING_SIZE(info) / 2;
975 	} else
976 		info->tag_set.queue_depth = BLK_RING_SIZE(info);
977 	info->tag_set.numa_node = NUMA_NO_NODE;
978 	info->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE;
979 	info->tag_set.cmd_size = sizeof(struct blkif_req);
980 	info->tag_set.driver_data = info;
981 
982 	if (blk_mq_alloc_tag_set(&info->tag_set))
983 		return -EINVAL;
984 	rq = blk_mq_init_queue(&info->tag_set);
985 	if (IS_ERR(rq)) {
986 		blk_mq_free_tag_set(&info->tag_set);
987 		return PTR_ERR(rq);
988 	}
989 
990 	rq->queuedata = info;
991 	info->rq = gd->queue = rq;
992 	info->gd = gd;
993 	info->sector_size = sector_size;
994 	info->physical_sector_size = physical_sector_size;
995 	blkif_set_queue_limits(info);
996 
997 	return 0;
998 }
999 
1000 static const char *flush_info(struct blkfront_info *info)
1001 {
1002 	if (info->feature_flush && info->feature_fua)
1003 		return "barrier: enabled;";
1004 	else if (info->feature_flush)
1005 		return "flush diskcache: enabled;";
1006 	else
1007 		return "barrier or flush: disabled;";
1008 }
1009 
1010 static void xlvbd_flush(struct blkfront_info *info)
1011 {
1012 	blk_queue_write_cache(info->rq, info->feature_flush ? true : false,
1013 			      info->feature_fua ? true : false);
1014 	pr_info("blkfront: %s: %s %s %s %s %s\n",
1015 		info->gd->disk_name, flush_info(info),
1016 		"persistent grants:", info->feature_persistent ?
1017 		"enabled;" : "disabled;", "indirect descriptors:",
1018 		info->max_indirect_segments ? "enabled;" : "disabled;");
1019 }
1020 
1021 static int xen_translate_vdev(int vdevice, int *minor, unsigned int *offset)
1022 {
1023 	int major;
1024 	major = BLKIF_MAJOR(vdevice);
1025 	*minor = BLKIF_MINOR(vdevice);
1026 	switch (major) {
1027 		case XEN_IDE0_MAJOR:
1028 			*offset = (*minor / 64) + EMULATED_HD_DISK_NAME_OFFSET;
1029 			*minor = ((*minor / 64) * PARTS_PER_DISK) +
1030 				EMULATED_HD_DISK_MINOR_OFFSET;
1031 			break;
1032 		case XEN_IDE1_MAJOR:
1033 			*offset = (*minor / 64) + 2 + EMULATED_HD_DISK_NAME_OFFSET;
1034 			*minor = (((*minor / 64) + 2) * PARTS_PER_DISK) +
1035 				EMULATED_HD_DISK_MINOR_OFFSET;
1036 			break;
1037 		case XEN_SCSI_DISK0_MAJOR:
1038 			*offset = (*minor / PARTS_PER_DISK) + EMULATED_SD_DISK_NAME_OFFSET;
1039 			*minor = *minor + EMULATED_SD_DISK_MINOR_OFFSET;
1040 			break;
1041 		case XEN_SCSI_DISK1_MAJOR:
1042 		case XEN_SCSI_DISK2_MAJOR:
1043 		case XEN_SCSI_DISK3_MAJOR:
1044 		case XEN_SCSI_DISK4_MAJOR:
1045 		case XEN_SCSI_DISK5_MAJOR:
1046 		case XEN_SCSI_DISK6_MAJOR:
1047 		case XEN_SCSI_DISK7_MAJOR:
1048 			*offset = (*minor / PARTS_PER_DISK) +
1049 				((major - XEN_SCSI_DISK1_MAJOR + 1) * 16) +
1050 				EMULATED_SD_DISK_NAME_OFFSET;
1051 			*minor = *minor +
1052 				((major - XEN_SCSI_DISK1_MAJOR + 1) * 16 * PARTS_PER_DISK) +
1053 				EMULATED_SD_DISK_MINOR_OFFSET;
1054 			break;
1055 		case XEN_SCSI_DISK8_MAJOR:
1056 		case XEN_SCSI_DISK9_MAJOR:
1057 		case XEN_SCSI_DISK10_MAJOR:
1058 		case XEN_SCSI_DISK11_MAJOR:
1059 		case XEN_SCSI_DISK12_MAJOR:
1060 		case XEN_SCSI_DISK13_MAJOR:
1061 		case XEN_SCSI_DISK14_MAJOR:
1062 		case XEN_SCSI_DISK15_MAJOR:
1063 			*offset = (*minor / PARTS_PER_DISK) +
1064 				((major - XEN_SCSI_DISK8_MAJOR + 8) * 16) +
1065 				EMULATED_SD_DISK_NAME_OFFSET;
1066 			*minor = *minor +
1067 				((major - XEN_SCSI_DISK8_MAJOR + 8) * 16 * PARTS_PER_DISK) +
1068 				EMULATED_SD_DISK_MINOR_OFFSET;
1069 			break;
1070 		case XENVBD_MAJOR:
1071 			*offset = *minor / PARTS_PER_DISK;
1072 			break;
1073 		default:
1074 			printk(KERN_WARNING "blkfront: your disk configuration is "
1075 					"incorrect, please use an xvd device instead\n");
1076 			return -ENODEV;
1077 	}
1078 	return 0;
1079 }
1080 
1081 static char *encode_disk_name(char *ptr, unsigned int n)
1082 {
1083 	if (n >= 26)
1084 		ptr = encode_disk_name(ptr, n / 26 - 1);
1085 	*ptr = 'a' + n % 26;
1086 	return ptr + 1;
1087 }
1088 
1089 static int xlvbd_alloc_gendisk(blkif_sector_t capacity,
1090 			       struct blkfront_info *info,
1091 			       u16 vdisk_info, u16 sector_size,
1092 			       unsigned int physical_sector_size)
1093 {
1094 	struct gendisk *gd;
1095 	int nr_minors = 1;
1096 	int err;
1097 	unsigned int offset;
1098 	int minor;
1099 	int nr_parts;
1100 	char *ptr;
1101 
1102 	BUG_ON(info->gd != NULL);
1103 	BUG_ON(info->rq != NULL);
1104 
1105 	if ((info->vdevice>>EXT_SHIFT) > 1) {
1106 		/* this is above the extended range; something is wrong */
1107 		printk(KERN_WARNING "blkfront: vdevice 0x%x is above the extended range; ignoring\n", info->vdevice);
1108 		return -ENODEV;
1109 	}
1110 
1111 	if (!VDEV_IS_EXTENDED(info->vdevice)) {
1112 		err = xen_translate_vdev(info->vdevice, &minor, &offset);
1113 		if (err)
1114 			return err;
1115  		nr_parts = PARTS_PER_DISK;
1116 	} else {
1117 		minor = BLKIF_MINOR_EXT(info->vdevice);
1118 		nr_parts = PARTS_PER_EXT_DISK;
1119 		offset = minor / nr_parts;
1120 		if (xen_hvm_domain() && offset < EMULATED_HD_DISK_NAME_OFFSET + 4)
1121 			printk(KERN_WARNING "blkfront: vdevice 0x%x might conflict with "
1122 					"emulated IDE disks,\n\t choose an xvd device name"
1123 					"from xvde on\n", info->vdevice);
1124 	}
1125 	if (minor >> MINORBITS) {
1126 		pr_warn("blkfront: %#x's minor (%#x) out of range; ignoring\n",
1127 			info->vdevice, minor);
1128 		return -ENODEV;
1129 	}
1130 
1131 	if ((minor % nr_parts) == 0)
1132 		nr_minors = nr_parts;
1133 
1134 	err = xlbd_reserve_minors(minor, nr_minors);
1135 	if (err)
1136 		goto out;
1137 	err = -ENODEV;
1138 
1139 	gd = alloc_disk(nr_minors);
1140 	if (gd == NULL)
1141 		goto release;
1142 
1143 	strcpy(gd->disk_name, DEV_NAME);
1144 	ptr = encode_disk_name(gd->disk_name + sizeof(DEV_NAME) - 1, offset);
1145 	BUG_ON(ptr >= gd->disk_name + DISK_NAME_LEN);
1146 	if (nr_minors > 1)
1147 		*ptr = 0;
1148 	else
1149 		snprintf(ptr, gd->disk_name + DISK_NAME_LEN - ptr,
1150 			 "%d", minor & (nr_parts - 1));
1151 
1152 	gd->major = XENVBD_MAJOR;
1153 	gd->first_minor = minor;
1154 	gd->fops = &xlvbd_block_fops;
1155 	gd->private_data = info;
1156 	set_capacity(gd, capacity);
1157 
1158 	if (xlvbd_init_blk_queue(gd, sector_size, physical_sector_size)) {
1159 		del_gendisk(gd);
1160 		goto release;
1161 	}
1162 
1163 	xlvbd_flush(info);
1164 
1165 	if (vdisk_info & VDISK_READONLY)
1166 		set_disk_ro(gd, 1);
1167 
1168 	if (vdisk_info & VDISK_REMOVABLE)
1169 		gd->flags |= GENHD_FL_REMOVABLE;
1170 
1171 	if (vdisk_info & VDISK_CDROM)
1172 		gd->flags |= GENHD_FL_CD;
1173 
1174 	return 0;
1175 
1176  release:
1177 	xlbd_release_minors(minor, nr_minors);
1178  out:
1179 	return err;
1180 }
1181 
1182 static void xlvbd_release_gendisk(struct blkfront_info *info)
1183 {
1184 	unsigned int minor, nr_minors, i;
1185 
1186 	if (info->rq == NULL)
1187 		return;
1188 
1189 	/* No more blkif_request(). */
1190 	blk_mq_stop_hw_queues(info->rq);
1191 
1192 	for (i = 0; i < info->nr_rings; i++) {
1193 		struct blkfront_ring_info *rinfo = &info->rinfo[i];
1194 
1195 		/* No more gnttab callback work. */
1196 		gnttab_cancel_free_callback(&rinfo->callback);
1197 
1198 		/* Flush gnttab callback work. Must be done with no locks held. */
1199 		flush_work(&rinfo->work);
1200 	}
1201 
1202 	del_gendisk(info->gd);
1203 
1204 	minor = info->gd->first_minor;
1205 	nr_minors = info->gd->minors;
1206 	xlbd_release_minors(minor, nr_minors);
1207 
1208 	blk_cleanup_queue(info->rq);
1209 	blk_mq_free_tag_set(&info->tag_set);
1210 	info->rq = NULL;
1211 
1212 	put_disk(info->gd);
1213 	info->gd = NULL;
1214 }
1215 
1216 /* Already hold rinfo->ring_lock. */
1217 static inline void kick_pending_request_queues_locked(struct blkfront_ring_info *rinfo)
1218 {
1219 	if (!RING_FULL(&rinfo->ring))
1220 		blk_mq_start_stopped_hw_queues(rinfo->dev_info->rq, true);
1221 }
1222 
1223 static void kick_pending_request_queues(struct blkfront_ring_info *rinfo)
1224 {
1225 	unsigned long flags;
1226 
1227 	spin_lock_irqsave(&rinfo->ring_lock, flags);
1228 	kick_pending_request_queues_locked(rinfo);
1229 	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
1230 }
1231 
1232 static void blkif_restart_queue(struct work_struct *work)
1233 {
1234 	struct blkfront_ring_info *rinfo = container_of(work, struct blkfront_ring_info, work);
1235 
1236 	if (rinfo->dev_info->connected == BLKIF_STATE_CONNECTED)
1237 		kick_pending_request_queues(rinfo);
1238 }
1239 
1240 static void blkif_free_ring(struct blkfront_ring_info *rinfo)
1241 {
1242 	struct grant *persistent_gnt, *n;
1243 	struct blkfront_info *info = rinfo->dev_info;
1244 	int i, j, segs;
1245 
1246 	/*
1247 	 * Remove indirect pages, this only happens when using indirect
1248 	 * descriptors but not persistent grants
1249 	 */
1250 	if (!list_empty(&rinfo->indirect_pages)) {
1251 		struct page *indirect_page, *n;
1252 
1253 		BUG_ON(info->feature_persistent);
1254 		list_for_each_entry_safe(indirect_page, n, &rinfo->indirect_pages, lru) {
1255 			list_del(&indirect_page->lru);
1256 			__free_page(indirect_page);
1257 		}
1258 	}
1259 
1260 	/* Remove all persistent grants. */
1261 	if (!list_empty(&rinfo->grants)) {
1262 		list_for_each_entry_safe(persistent_gnt, n,
1263 					 &rinfo->grants, node) {
1264 			list_del(&persistent_gnt->node);
1265 			if (persistent_gnt->gref != GRANT_INVALID_REF) {
1266 				gnttab_end_foreign_access(persistent_gnt->gref,
1267 							  0, 0UL);
1268 				rinfo->persistent_gnts_c--;
1269 			}
1270 			if (info->feature_persistent)
1271 				__free_page(persistent_gnt->page);
1272 			kfree(persistent_gnt);
1273 		}
1274 	}
1275 	BUG_ON(rinfo->persistent_gnts_c != 0);
1276 
1277 	for (i = 0; i < BLK_RING_SIZE(info); i++) {
1278 		/*
1279 		 * Clear persistent grants present in requests already
1280 		 * on the shared ring
1281 		 */
1282 		if (!rinfo->shadow[i].request)
1283 			goto free_shadow;
1284 
1285 		segs = rinfo->shadow[i].req.operation == BLKIF_OP_INDIRECT ?
1286 		       rinfo->shadow[i].req.u.indirect.nr_segments :
1287 		       rinfo->shadow[i].req.u.rw.nr_segments;
1288 		for (j = 0; j < segs; j++) {
1289 			persistent_gnt = rinfo->shadow[i].grants_used[j];
1290 			gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL);
1291 			if (info->feature_persistent)
1292 				__free_page(persistent_gnt->page);
1293 			kfree(persistent_gnt);
1294 		}
1295 
1296 		if (rinfo->shadow[i].req.operation != BLKIF_OP_INDIRECT)
1297 			/*
1298 			 * If this is not an indirect operation don't try to
1299 			 * free indirect segments
1300 			 */
1301 			goto free_shadow;
1302 
1303 		for (j = 0; j < INDIRECT_GREFS(segs); j++) {
1304 			persistent_gnt = rinfo->shadow[i].indirect_grants[j];
1305 			gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL);
1306 			__free_page(persistent_gnt->page);
1307 			kfree(persistent_gnt);
1308 		}
1309 
1310 free_shadow:
1311 		kfree(rinfo->shadow[i].grants_used);
1312 		rinfo->shadow[i].grants_used = NULL;
1313 		kfree(rinfo->shadow[i].indirect_grants);
1314 		rinfo->shadow[i].indirect_grants = NULL;
1315 		kfree(rinfo->shadow[i].sg);
1316 		rinfo->shadow[i].sg = NULL;
1317 	}
1318 
1319 	/* No more gnttab callback work. */
1320 	gnttab_cancel_free_callback(&rinfo->callback);
1321 
1322 	/* Flush gnttab callback work. Must be done with no locks held. */
1323 	flush_work(&rinfo->work);
1324 
1325 	/* Free resources associated with old device channel. */
1326 	for (i = 0; i < info->nr_ring_pages; i++) {
1327 		if (rinfo->ring_ref[i] != GRANT_INVALID_REF) {
1328 			gnttab_end_foreign_access(rinfo->ring_ref[i], 0, 0);
1329 			rinfo->ring_ref[i] = GRANT_INVALID_REF;
1330 		}
1331 	}
1332 	free_pages((unsigned long)rinfo->ring.sring, get_order(info->nr_ring_pages * XEN_PAGE_SIZE));
1333 	rinfo->ring.sring = NULL;
1334 
1335 	if (rinfo->irq)
1336 		unbind_from_irqhandler(rinfo->irq, rinfo);
1337 	rinfo->evtchn = rinfo->irq = 0;
1338 }
1339 
1340 static void blkif_free(struct blkfront_info *info, int suspend)
1341 {
1342 	unsigned int i;
1343 
1344 	/* Prevent new requests being issued until we fix things up. */
1345 	info->connected = suspend ?
1346 		BLKIF_STATE_SUSPENDED : BLKIF_STATE_DISCONNECTED;
1347 	/* No more blkif_request(). */
1348 	if (info->rq)
1349 		blk_mq_stop_hw_queues(info->rq);
1350 
1351 	for (i = 0; i < info->nr_rings; i++)
1352 		blkif_free_ring(&info->rinfo[i]);
1353 
1354 	kfree(info->rinfo);
1355 	info->rinfo = NULL;
1356 	info->nr_rings = 0;
1357 }
1358 
1359 struct copy_from_grant {
1360 	const struct blk_shadow *s;
1361 	unsigned int grant_idx;
1362 	unsigned int bvec_offset;
1363 	char *bvec_data;
1364 };
1365 
1366 static void blkif_copy_from_grant(unsigned long gfn, unsigned int offset,
1367 				  unsigned int len, void *data)
1368 {
1369 	struct copy_from_grant *info = data;
1370 	char *shared_data;
1371 	/* Convenient aliases */
1372 	const struct blk_shadow *s = info->s;
1373 
1374 	shared_data = kmap_atomic(s->grants_used[info->grant_idx]->page);
1375 
1376 	memcpy(info->bvec_data + info->bvec_offset,
1377 	       shared_data + offset, len);
1378 
1379 	info->bvec_offset += len;
1380 	info->grant_idx++;
1381 
1382 	kunmap_atomic(shared_data);
1383 }
1384 
1385 static enum blk_req_status blkif_rsp_to_req_status(int rsp)
1386 {
1387 	switch (rsp)
1388 	{
1389 	case BLKIF_RSP_OKAY:
1390 		return REQ_DONE;
1391 	case BLKIF_RSP_EOPNOTSUPP:
1392 		return REQ_EOPNOTSUPP;
1393 	case BLKIF_RSP_ERROR:
1394 		/* Fallthrough. */
1395 	default:
1396 		return REQ_ERROR;
1397 	}
1398 }
1399 
1400 /*
1401  * Get the final status of the block request based on two ring response
1402  */
1403 static int blkif_get_final_status(enum blk_req_status s1,
1404 				  enum blk_req_status s2)
1405 {
1406 	BUG_ON(s1 == REQ_WAITING);
1407 	BUG_ON(s2 == REQ_WAITING);
1408 
1409 	if (s1 == REQ_ERROR || s2 == REQ_ERROR)
1410 		return BLKIF_RSP_ERROR;
1411 	else if (s1 == REQ_EOPNOTSUPP || s2 == REQ_EOPNOTSUPP)
1412 		return BLKIF_RSP_EOPNOTSUPP;
1413 	return BLKIF_RSP_OKAY;
1414 }
1415 
1416 static bool blkif_completion(unsigned long *id,
1417 			     struct blkfront_ring_info *rinfo,
1418 			     struct blkif_response *bret)
1419 {
1420 	int i = 0;
1421 	struct scatterlist *sg;
1422 	int num_sg, num_grant;
1423 	struct blkfront_info *info = rinfo->dev_info;
1424 	struct blk_shadow *s = &rinfo->shadow[*id];
1425 	struct copy_from_grant data = {
1426 		.grant_idx = 0,
1427 	};
1428 
1429 	num_grant = s->req.operation == BLKIF_OP_INDIRECT ?
1430 		s->req.u.indirect.nr_segments : s->req.u.rw.nr_segments;
1431 
1432 	/* The I/O request may be split in two. */
1433 	if (unlikely(s->associated_id != NO_ASSOCIATED_ID)) {
1434 		struct blk_shadow *s2 = &rinfo->shadow[s->associated_id];
1435 
1436 		/* Keep the status of the current response in shadow. */
1437 		s->status = blkif_rsp_to_req_status(bret->status);
1438 
1439 		/* Wait the second response if not yet here. */
1440 		if (s2->status == REQ_WAITING)
1441 			return 0;
1442 
1443 		bret->status = blkif_get_final_status(s->status,
1444 						      s2->status);
1445 
1446 		/*
1447 		 * All the grants is stored in the first shadow in order
1448 		 * to make the completion code simpler.
1449 		 */
1450 		num_grant += s2->req.u.rw.nr_segments;
1451 
1452 		/*
1453 		 * The two responses may not come in order. Only the
1454 		 * first request will store the scatter-gather list.
1455 		 */
1456 		if (s2->num_sg != 0) {
1457 			/* Update "id" with the ID of the first response. */
1458 			*id = s->associated_id;
1459 			s = s2;
1460 		}
1461 
1462 		/*
1463 		 * We don't need anymore the second request, so recycling
1464 		 * it now.
1465 		 */
1466 		if (add_id_to_freelist(rinfo, s->associated_id))
1467 			WARN(1, "%s: can't recycle the second part (id = %ld) of the request\n",
1468 			     info->gd->disk_name, s->associated_id);
1469 	}
1470 
1471 	data.s = s;
1472 	num_sg = s->num_sg;
1473 
1474 	if (bret->operation == BLKIF_OP_READ && info->feature_persistent) {
1475 		for_each_sg(s->sg, sg, num_sg, i) {
1476 			BUG_ON(sg->offset + sg->length > PAGE_SIZE);
1477 
1478 			data.bvec_offset = sg->offset;
1479 			data.bvec_data = kmap_atomic(sg_page(sg));
1480 
1481 			gnttab_foreach_grant_in_range(sg_page(sg),
1482 						      sg->offset,
1483 						      sg->length,
1484 						      blkif_copy_from_grant,
1485 						      &data);
1486 
1487 			kunmap_atomic(data.bvec_data);
1488 		}
1489 	}
1490 	/* Add the persistent grant into the list of free grants */
1491 	for (i = 0; i < num_grant; i++) {
1492 		if (gnttab_query_foreign_access(s->grants_used[i]->gref)) {
1493 			/*
1494 			 * If the grant is still mapped by the backend (the
1495 			 * backend has chosen to make this grant persistent)
1496 			 * we add it at the head of the list, so it will be
1497 			 * reused first.
1498 			 */
1499 			if (!info->feature_persistent)
1500 				pr_alert_ratelimited("backed has not unmapped grant: %u\n",
1501 						     s->grants_used[i]->gref);
1502 			list_add(&s->grants_used[i]->node, &rinfo->grants);
1503 			rinfo->persistent_gnts_c++;
1504 		} else {
1505 			/*
1506 			 * If the grant is not mapped by the backend we end the
1507 			 * foreign access and add it to the tail of the list,
1508 			 * so it will not be picked again unless we run out of
1509 			 * persistent grants.
1510 			 */
1511 			gnttab_end_foreign_access(s->grants_used[i]->gref, 0, 0UL);
1512 			s->grants_used[i]->gref = GRANT_INVALID_REF;
1513 			list_add_tail(&s->grants_used[i]->node, &rinfo->grants);
1514 		}
1515 	}
1516 	if (s->req.operation == BLKIF_OP_INDIRECT) {
1517 		for (i = 0; i < INDIRECT_GREFS(num_grant); i++) {
1518 			if (gnttab_query_foreign_access(s->indirect_grants[i]->gref)) {
1519 				if (!info->feature_persistent)
1520 					pr_alert_ratelimited("backed has not unmapped grant: %u\n",
1521 							     s->indirect_grants[i]->gref);
1522 				list_add(&s->indirect_grants[i]->node, &rinfo->grants);
1523 				rinfo->persistent_gnts_c++;
1524 			} else {
1525 				struct page *indirect_page;
1526 
1527 				gnttab_end_foreign_access(s->indirect_grants[i]->gref, 0, 0UL);
1528 				/*
1529 				 * Add the used indirect page back to the list of
1530 				 * available pages for indirect grefs.
1531 				 */
1532 				if (!info->feature_persistent) {
1533 					indirect_page = s->indirect_grants[i]->page;
1534 					list_add(&indirect_page->lru, &rinfo->indirect_pages);
1535 				}
1536 				s->indirect_grants[i]->gref = GRANT_INVALID_REF;
1537 				list_add_tail(&s->indirect_grants[i]->node, &rinfo->grants);
1538 			}
1539 		}
1540 	}
1541 
1542 	return 1;
1543 }
1544 
1545 static irqreturn_t blkif_interrupt(int irq, void *dev_id)
1546 {
1547 	struct request *req;
1548 	struct blkif_response *bret;
1549 	RING_IDX i, rp;
1550 	unsigned long flags;
1551 	struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)dev_id;
1552 	struct blkfront_info *info = rinfo->dev_info;
1553 
1554 	if (unlikely(info->connected != BLKIF_STATE_CONNECTED))
1555 		return IRQ_HANDLED;
1556 
1557 	spin_lock_irqsave(&rinfo->ring_lock, flags);
1558  again:
1559 	rp = rinfo->ring.sring->rsp_prod;
1560 	rmb(); /* Ensure we see queued responses up to 'rp'. */
1561 
1562 	for (i = rinfo->ring.rsp_cons; i != rp; i++) {
1563 		unsigned long id;
1564 
1565 		bret = RING_GET_RESPONSE(&rinfo->ring, i);
1566 		id   = bret->id;
1567 		/*
1568 		 * The backend has messed up and given us an id that we would
1569 		 * never have given to it (we stamp it up to BLK_RING_SIZE -
1570 		 * look in get_id_from_freelist.
1571 		 */
1572 		if (id >= BLK_RING_SIZE(info)) {
1573 			WARN(1, "%s: response to %s has incorrect id (%ld)\n",
1574 			     info->gd->disk_name, op_name(bret->operation), id);
1575 			/* We can't safely get the 'struct request' as
1576 			 * the id is busted. */
1577 			continue;
1578 		}
1579 		req  = rinfo->shadow[id].request;
1580 
1581 		if (bret->operation != BLKIF_OP_DISCARD) {
1582 			/*
1583 			 * We may need to wait for an extra response if the
1584 			 * I/O request is split in 2
1585 			 */
1586 			if (!blkif_completion(&id, rinfo, bret))
1587 				continue;
1588 		}
1589 
1590 		if (add_id_to_freelist(rinfo, id)) {
1591 			WARN(1, "%s: response to %s (id %ld) couldn't be recycled!\n",
1592 			     info->gd->disk_name, op_name(bret->operation), id);
1593 			continue;
1594 		}
1595 
1596 		if (bret->status == BLKIF_RSP_OKAY)
1597 			blkif_req(req)->error = BLK_STS_OK;
1598 		else
1599 			blkif_req(req)->error = BLK_STS_IOERR;
1600 
1601 		switch (bret->operation) {
1602 		case BLKIF_OP_DISCARD:
1603 			if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) {
1604 				struct request_queue *rq = info->rq;
1605 				printk(KERN_WARNING "blkfront: %s: %s op failed\n",
1606 					   info->gd->disk_name, op_name(bret->operation));
1607 				blkif_req(req)->error = BLK_STS_NOTSUPP;
1608 				info->feature_discard = 0;
1609 				info->feature_secdiscard = 0;
1610 				queue_flag_clear(QUEUE_FLAG_DISCARD, rq);
1611 				queue_flag_clear(QUEUE_FLAG_SECERASE, rq);
1612 			}
1613 			break;
1614 		case BLKIF_OP_FLUSH_DISKCACHE:
1615 		case BLKIF_OP_WRITE_BARRIER:
1616 			if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) {
1617 				printk(KERN_WARNING "blkfront: %s: %s op failed\n",
1618 				       info->gd->disk_name, op_name(bret->operation));
1619 				blkif_req(req)->error = -EOPNOTSUPP;
1620 			}
1621 			if (unlikely(bret->status == BLKIF_RSP_ERROR &&
1622 				     rinfo->shadow[id].req.u.rw.nr_segments == 0)) {
1623 				printk(KERN_WARNING "blkfront: %s: empty %s op failed\n",
1624 				       info->gd->disk_name, op_name(bret->operation));
1625 				blkif_req(req)->error = BLK_STS_NOTSUPP;
1626 			}
1627 			if (unlikely(blkif_req(req)->error)) {
1628 				if (blkif_req(req)->error == BLK_STS_NOTSUPP)
1629 					blkif_req(req)->error = BLK_STS_OK;
1630 				info->feature_fua = 0;
1631 				info->feature_flush = 0;
1632 				xlvbd_flush(info);
1633 			}
1634 			/* fall through */
1635 		case BLKIF_OP_READ:
1636 		case BLKIF_OP_WRITE:
1637 			if (unlikely(bret->status != BLKIF_RSP_OKAY))
1638 				dev_dbg(&info->xbdev->dev, "Bad return from blkdev data "
1639 					"request: %x\n", bret->status);
1640 
1641 			break;
1642 		default:
1643 			BUG();
1644 		}
1645 
1646 		blk_mq_complete_request(req);
1647 	}
1648 
1649 	rinfo->ring.rsp_cons = i;
1650 
1651 	if (i != rinfo->ring.req_prod_pvt) {
1652 		int more_to_do;
1653 		RING_FINAL_CHECK_FOR_RESPONSES(&rinfo->ring, more_to_do);
1654 		if (more_to_do)
1655 			goto again;
1656 	} else
1657 		rinfo->ring.sring->rsp_event = i + 1;
1658 
1659 	kick_pending_request_queues_locked(rinfo);
1660 
1661 	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
1662 
1663 	return IRQ_HANDLED;
1664 }
1665 
1666 
1667 static int setup_blkring(struct xenbus_device *dev,
1668 			 struct blkfront_ring_info *rinfo)
1669 {
1670 	struct blkif_sring *sring;
1671 	int err, i;
1672 	struct blkfront_info *info = rinfo->dev_info;
1673 	unsigned long ring_size = info->nr_ring_pages * XEN_PAGE_SIZE;
1674 	grant_ref_t gref[XENBUS_MAX_RING_GRANTS];
1675 
1676 	for (i = 0; i < info->nr_ring_pages; i++)
1677 		rinfo->ring_ref[i] = GRANT_INVALID_REF;
1678 
1679 	sring = (struct blkif_sring *)__get_free_pages(GFP_NOIO | __GFP_HIGH,
1680 						       get_order(ring_size));
1681 	if (!sring) {
1682 		xenbus_dev_fatal(dev, -ENOMEM, "allocating shared ring");
1683 		return -ENOMEM;
1684 	}
1685 	SHARED_RING_INIT(sring);
1686 	FRONT_RING_INIT(&rinfo->ring, sring, ring_size);
1687 
1688 	err = xenbus_grant_ring(dev, rinfo->ring.sring, info->nr_ring_pages, gref);
1689 	if (err < 0) {
1690 		free_pages((unsigned long)sring, get_order(ring_size));
1691 		rinfo->ring.sring = NULL;
1692 		goto fail;
1693 	}
1694 	for (i = 0; i < info->nr_ring_pages; i++)
1695 		rinfo->ring_ref[i] = gref[i];
1696 
1697 	err = xenbus_alloc_evtchn(dev, &rinfo->evtchn);
1698 	if (err)
1699 		goto fail;
1700 
1701 	err = bind_evtchn_to_irqhandler(rinfo->evtchn, blkif_interrupt, 0,
1702 					"blkif", rinfo);
1703 	if (err <= 0) {
1704 		xenbus_dev_fatal(dev, err,
1705 				 "bind_evtchn_to_irqhandler failed");
1706 		goto fail;
1707 	}
1708 	rinfo->irq = err;
1709 
1710 	return 0;
1711 fail:
1712 	blkif_free(info, 0);
1713 	return err;
1714 }
1715 
1716 /*
1717  * Write out per-ring/queue nodes including ring-ref and event-channel, and each
1718  * ring buffer may have multi pages depending on ->nr_ring_pages.
1719  */
1720 static int write_per_ring_nodes(struct xenbus_transaction xbt,
1721 				struct blkfront_ring_info *rinfo, const char *dir)
1722 {
1723 	int err;
1724 	unsigned int i;
1725 	const char *message = NULL;
1726 	struct blkfront_info *info = rinfo->dev_info;
1727 
1728 	if (info->nr_ring_pages == 1) {
1729 		err = xenbus_printf(xbt, dir, "ring-ref", "%u", rinfo->ring_ref[0]);
1730 		if (err) {
1731 			message = "writing ring-ref";
1732 			goto abort_transaction;
1733 		}
1734 	} else {
1735 		for (i = 0; i < info->nr_ring_pages; i++) {
1736 			char ring_ref_name[RINGREF_NAME_LEN];
1737 
1738 			snprintf(ring_ref_name, RINGREF_NAME_LEN, "ring-ref%u", i);
1739 			err = xenbus_printf(xbt, dir, ring_ref_name,
1740 					    "%u", rinfo->ring_ref[i]);
1741 			if (err) {
1742 				message = "writing ring-ref";
1743 				goto abort_transaction;
1744 			}
1745 		}
1746 	}
1747 
1748 	err = xenbus_printf(xbt, dir, "event-channel", "%u", rinfo->evtchn);
1749 	if (err) {
1750 		message = "writing event-channel";
1751 		goto abort_transaction;
1752 	}
1753 
1754 	return 0;
1755 
1756 abort_transaction:
1757 	xenbus_transaction_end(xbt, 1);
1758 	if (message)
1759 		xenbus_dev_fatal(info->xbdev, err, "%s", message);
1760 
1761 	return err;
1762 }
1763 
1764 /* Common code used when first setting up, and when resuming. */
1765 static int talk_to_blkback(struct xenbus_device *dev,
1766 			   struct blkfront_info *info)
1767 {
1768 	const char *message = NULL;
1769 	struct xenbus_transaction xbt;
1770 	int err;
1771 	unsigned int i, max_page_order;
1772 	unsigned int ring_page_order;
1773 
1774 	max_page_order = xenbus_read_unsigned(info->xbdev->otherend,
1775 					      "max-ring-page-order", 0);
1776 	ring_page_order = min(xen_blkif_max_ring_order, max_page_order);
1777 	info->nr_ring_pages = 1 << ring_page_order;
1778 
1779 	for (i = 0; i < info->nr_rings; i++) {
1780 		struct blkfront_ring_info *rinfo = &info->rinfo[i];
1781 
1782 		/* Create shared ring, alloc event channel. */
1783 		err = setup_blkring(dev, rinfo);
1784 		if (err)
1785 			goto destroy_blkring;
1786 	}
1787 
1788 again:
1789 	err = xenbus_transaction_start(&xbt);
1790 	if (err) {
1791 		xenbus_dev_fatal(dev, err, "starting transaction");
1792 		goto destroy_blkring;
1793 	}
1794 
1795 	if (info->nr_ring_pages > 1) {
1796 		err = xenbus_printf(xbt, dev->nodename, "ring-page-order", "%u",
1797 				    ring_page_order);
1798 		if (err) {
1799 			message = "writing ring-page-order";
1800 			goto abort_transaction;
1801 		}
1802 	}
1803 
1804 	/* We already got the number of queues/rings in _probe */
1805 	if (info->nr_rings == 1) {
1806 		err = write_per_ring_nodes(xbt, &info->rinfo[0], dev->nodename);
1807 		if (err)
1808 			goto destroy_blkring;
1809 	} else {
1810 		char *path;
1811 		size_t pathsize;
1812 
1813 		err = xenbus_printf(xbt, dev->nodename, "multi-queue-num-queues", "%u",
1814 				    info->nr_rings);
1815 		if (err) {
1816 			message = "writing multi-queue-num-queues";
1817 			goto abort_transaction;
1818 		}
1819 
1820 		pathsize = strlen(dev->nodename) + QUEUE_NAME_LEN;
1821 		path = kmalloc(pathsize, GFP_KERNEL);
1822 		if (!path) {
1823 			err = -ENOMEM;
1824 			message = "ENOMEM while writing ring references";
1825 			goto abort_transaction;
1826 		}
1827 
1828 		for (i = 0; i < info->nr_rings; i++) {
1829 			memset(path, 0, pathsize);
1830 			snprintf(path, pathsize, "%s/queue-%u", dev->nodename, i);
1831 			err = write_per_ring_nodes(xbt, &info->rinfo[i], path);
1832 			if (err) {
1833 				kfree(path);
1834 				goto destroy_blkring;
1835 			}
1836 		}
1837 		kfree(path);
1838 	}
1839 	err = xenbus_printf(xbt, dev->nodename, "protocol", "%s",
1840 			    XEN_IO_PROTO_ABI_NATIVE);
1841 	if (err) {
1842 		message = "writing protocol";
1843 		goto abort_transaction;
1844 	}
1845 	err = xenbus_printf(xbt, dev->nodename,
1846 			    "feature-persistent", "%u", 1);
1847 	if (err)
1848 		dev_warn(&dev->dev,
1849 			 "writing persistent grants feature to xenbus");
1850 
1851 	err = xenbus_transaction_end(xbt, 0);
1852 	if (err) {
1853 		if (err == -EAGAIN)
1854 			goto again;
1855 		xenbus_dev_fatal(dev, err, "completing transaction");
1856 		goto destroy_blkring;
1857 	}
1858 
1859 	for (i = 0; i < info->nr_rings; i++) {
1860 		unsigned int j;
1861 		struct blkfront_ring_info *rinfo = &info->rinfo[i];
1862 
1863 		for (j = 0; j < BLK_RING_SIZE(info); j++)
1864 			rinfo->shadow[j].req.u.rw.id = j + 1;
1865 		rinfo->shadow[BLK_RING_SIZE(info)-1].req.u.rw.id = 0x0fffffff;
1866 	}
1867 	xenbus_switch_state(dev, XenbusStateInitialised);
1868 
1869 	return 0;
1870 
1871  abort_transaction:
1872 	xenbus_transaction_end(xbt, 1);
1873 	if (message)
1874 		xenbus_dev_fatal(dev, err, "%s", message);
1875  destroy_blkring:
1876 	blkif_free(info, 0);
1877 
1878 	kfree(info);
1879 	dev_set_drvdata(&dev->dev, NULL);
1880 
1881 	return err;
1882 }
1883 
1884 static int negotiate_mq(struct blkfront_info *info)
1885 {
1886 	unsigned int backend_max_queues;
1887 	unsigned int i;
1888 
1889 	BUG_ON(info->nr_rings);
1890 
1891 	/* Check if backend supports multiple queues. */
1892 	backend_max_queues = xenbus_read_unsigned(info->xbdev->otherend,
1893 						  "multi-queue-max-queues", 1);
1894 	info->nr_rings = min(backend_max_queues, xen_blkif_max_queues);
1895 	/* We need at least one ring. */
1896 	if (!info->nr_rings)
1897 		info->nr_rings = 1;
1898 
1899 	info->rinfo = kzalloc(sizeof(struct blkfront_ring_info) * info->nr_rings, GFP_KERNEL);
1900 	if (!info->rinfo) {
1901 		xenbus_dev_fatal(info->xbdev, -ENOMEM, "allocating ring_info structure");
1902 		return -ENOMEM;
1903 	}
1904 
1905 	for (i = 0; i < info->nr_rings; i++) {
1906 		struct blkfront_ring_info *rinfo;
1907 
1908 		rinfo = &info->rinfo[i];
1909 		INIT_LIST_HEAD(&rinfo->indirect_pages);
1910 		INIT_LIST_HEAD(&rinfo->grants);
1911 		rinfo->dev_info = info;
1912 		INIT_WORK(&rinfo->work, blkif_restart_queue);
1913 		spin_lock_init(&rinfo->ring_lock);
1914 	}
1915 	return 0;
1916 }
1917 /**
1918  * Entry point to this code when a new device is created.  Allocate the basic
1919  * structures and the ring buffer for communication with the backend, and
1920  * inform the backend of the appropriate details for those.  Switch to
1921  * Initialised state.
1922  */
1923 static int blkfront_probe(struct xenbus_device *dev,
1924 			  const struct xenbus_device_id *id)
1925 {
1926 	int err, vdevice;
1927 	struct blkfront_info *info;
1928 
1929 	/* FIXME: Use dynamic device id if this is not set. */
1930 	err = xenbus_scanf(XBT_NIL, dev->nodename,
1931 			   "virtual-device", "%i", &vdevice);
1932 	if (err != 1) {
1933 		/* go looking in the extended area instead */
1934 		err = xenbus_scanf(XBT_NIL, dev->nodename, "virtual-device-ext",
1935 				   "%i", &vdevice);
1936 		if (err != 1) {
1937 			xenbus_dev_fatal(dev, err, "reading virtual-device");
1938 			return err;
1939 		}
1940 	}
1941 
1942 	if (xen_hvm_domain()) {
1943 		char *type;
1944 		int len;
1945 		/* no unplug has been done: do not hook devices != xen vbds */
1946 		if (xen_has_pv_and_legacy_disk_devices()) {
1947 			int major;
1948 
1949 			if (!VDEV_IS_EXTENDED(vdevice))
1950 				major = BLKIF_MAJOR(vdevice);
1951 			else
1952 				major = XENVBD_MAJOR;
1953 
1954 			if (major != XENVBD_MAJOR) {
1955 				printk(KERN_INFO
1956 						"%s: HVM does not support vbd %d as xen block device\n",
1957 						__func__, vdevice);
1958 				return -ENODEV;
1959 			}
1960 		}
1961 		/* do not create a PV cdrom device if we are an HVM guest */
1962 		type = xenbus_read(XBT_NIL, dev->nodename, "device-type", &len);
1963 		if (IS_ERR(type))
1964 			return -ENODEV;
1965 		if (strncmp(type, "cdrom", 5) == 0) {
1966 			kfree(type);
1967 			return -ENODEV;
1968 		}
1969 		kfree(type);
1970 	}
1971 	info = kzalloc(sizeof(*info), GFP_KERNEL);
1972 	if (!info) {
1973 		xenbus_dev_fatal(dev, -ENOMEM, "allocating info structure");
1974 		return -ENOMEM;
1975 	}
1976 
1977 	info->xbdev = dev;
1978 	err = negotiate_mq(info);
1979 	if (err) {
1980 		kfree(info);
1981 		return err;
1982 	}
1983 
1984 	mutex_init(&info->mutex);
1985 	info->vdevice = vdevice;
1986 	info->connected = BLKIF_STATE_DISCONNECTED;
1987 
1988 	/* Front end dir is a number, which is used as the id. */
1989 	info->handle = simple_strtoul(strrchr(dev->nodename, '/')+1, NULL, 0);
1990 	dev_set_drvdata(&dev->dev, info);
1991 
1992 	return 0;
1993 }
1994 
1995 static int blkif_recover(struct blkfront_info *info)
1996 {
1997 	unsigned int r_index;
1998 	struct request *req, *n;
1999 	int rc;
2000 	struct bio *bio;
2001 	unsigned int segs;
2002 
2003 	blkfront_gather_backend_features(info);
2004 	/* Reset limits changed by blk_mq_update_nr_hw_queues(). */
2005 	blkif_set_queue_limits(info);
2006 	segs = info->max_indirect_segments ? : BLKIF_MAX_SEGMENTS_PER_REQUEST;
2007 	blk_queue_max_segments(info->rq, segs / GRANTS_PER_PSEG);
2008 
2009 	for (r_index = 0; r_index < info->nr_rings; r_index++) {
2010 		struct blkfront_ring_info *rinfo = &info->rinfo[r_index];
2011 
2012 		rc = blkfront_setup_indirect(rinfo);
2013 		if (rc)
2014 			return rc;
2015 	}
2016 	xenbus_switch_state(info->xbdev, XenbusStateConnected);
2017 
2018 	/* Now safe for us to use the shared ring */
2019 	info->connected = BLKIF_STATE_CONNECTED;
2020 
2021 	for (r_index = 0; r_index < info->nr_rings; r_index++) {
2022 		struct blkfront_ring_info *rinfo;
2023 
2024 		rinfo = &info->rinfo[r_index];
2025 		/* Kick any other new requests queued since we resumed */
2026 		kick_pending_request_queues(rinfo);
2027 	}
2028 
2029 	list_for_each_entry_safe(req, n, &info->requests, queuelist) {
2030 		/* Requeue pending requests (flush or discard) */
2031 		list_del_init(&req->queuelist);
2032 		BUG_ON(req->nr_phys_segments > segs);
2033 		blk_mq_requeue_request(req, false);
2034 	}
2035 	blk_mq_start_stopped_hw_queues(info->rq, true);
2036 	blk_mq_kick_requeue_list(info->rq);
2037 
2038 	while ((bio = bio_list_pop(&info->bio_list)) != NULL) {
2039 		/* Traverse the list of pending bios and re-queue them */
2040 		submit_bio(bio);
2041 	}
2042 
2043 	return 0;
2044 }
2045 
2046 /**
2047  * We are reconnecting to the backend, due to a suspend/resume, or a backend
2048  * driver restart.  We tear down our blkif structure and recreate it, but
2049  * leave the device-layer structures intact so that this is transparent to the
2050  * rest of the kernel.
2051  */
2052 static int blkfront_resume(struct xenbus_device *dev)
2053 {
2054 	struct blkfront_info *info = dev_get_drvdata(&dev->dev);
2055 	int err = 0;
2056 	unsigned int i, j;
2057 
2058 	dev_dbg(&dev->dev, "blkfront_resume: %s\n", dev->nodename);
2059 
2060 	bio_list_init(&info->bio_list);
2061 	INIT_LIST_HEAD(&info->requests);
2062 	for (i = 0; i < info->nr_rings; i++) {
2063 		struct blkfront_ring_info *rinfo = &info->rinfo[i];
2064 		struct bio_list merge_bio;
2065 		struct blk_shadow *shadow = rinfo->shadow;
2066 
2067 		for (j = 0; j < BLK_RING_SIZE(info); j++) {
2068 			/* Not in use? */
2069 			if (!shadow[j].request)
2070 				continue;
2071 
2072 			/*
2073 			 * Get the bios in the request so we can re-queue them.
2074 			 */
2075 			if (req_op(shadow[i].request) == REQ_OP_FLUSH ||
2076 			    req_op(shadow[i].request) == REQ_OP_DISCARD ||
2077 			    req_op(shadow[i].request) == REQ_OP_SECURE_ERASE ||
2078 			    shadow[j].request->cmd_flags & REQ_FUA) {
2079 				/*
2080 				 * Flush operations don't contain bios, so
2081 				 * we need to requeue the whole request
2082 				 *
2083 				 * XXX: but this doesn't make any sense for a
2084 				 * write with the FUA flag set..
2085 				 */
2086 				list_add(&shadow[j].request->queuelist, &info->requests);
2087 				continue;
2088 			}
2089 			merge_bio.head = shadow[j].request->bio;
2090 			merge_bio.tail = shadow[j].request->biotail;
2091 			bio_list_merge(&info->bio_list, &merge_bio);
2092 			shadow[j].request->bio = NULL;
2093 			blk_mq_end_request(shadow[j].request, BLK_STS_OK);
2094 		}
2095 	}
2096 
2097 	blkif_free(info, info->connected == BLKIF_STATE_CONNECTED);
2098 
2099 	err = negotiate_mq(info);
2100 	if (err)
2101 		return err;
2102 
2103 	err = talk_to_blkback(dev, info);
2104 	if (!err)
2105 		blk_mq_update_nr_hw_queues(&info->tag_set, info->nr_rings);
2106 
2107 	/*
2108 	 * We have to wait for the backend to switch to
2109 	 * connected state, since we want to read which
2110 	 * features it supports.
2111 	 */
2112 
2113 	return err;
2114 }
2115 
2116 static void blkfront_closing(struct blkfront_info *info)
2117 {
2118 	struct xenbus_device *xbdev = info->xbdev;
2119 	struct block_device *bdev = NULL;
2120 
2121 	mutex_lock(&info->mutex);
2122 
2123 	if (xbdev->state == XenbusStateClosing) {
2124 		mutex_unlock(&info->mutex);
2125 		return;
2126 	}
2127 
2128 	if (info->gd)
2129 		bdev = bdget_disk(info->gd, 0);
2130 
2131 	mutex_unlock(&info->mutex);
2132 
2133 	if (!bdev) {
2134 		xenbus_frontend_closed(xbdev);
2135 		return;
2136 	}
2137 
2138 	mutex_lock(&bdev->bd_mutex);
2139 
2140 	if (bdev->bd_openers) {
2141 		xenbus_dev_error(xbdev, -EBUSY,
2142 				 "Device in use; refusing to close");
2143 		xenbus_switch_state(xbdev, XenbusStateClosing);
2144 	} else {
2145 		xlvbd_release_gendisk(info);
2146 		xenbus_frontend_closed(xbdev);
2147 	}
2148 
2149 	mutex_unlock(&bdev->bd_mutex);
2150 	bdput(bdev);
2151 }
2152 
2153 static void blkfront_setup_discard(struct blkfront_info *info)
2154 {
2155 	int err;
2156 	unsigned int discard_granularity;
2157 	unsigned int discard_alignment;
2158 
2159 	info->feature_discard = 1;
2160 	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
2161 		"discard-granularity", "%u", &discard_granularity,
2162 		"discard-alignment", "%u", &discard_alignment,
2163 		NULL);
2164 	if (!err) {
2165 		info->discard_granularity = discard_granularity;
2166 		info->discard_alignment = discard_alignment;
2167 	}
2168 	info->feature_secdiscard =
2169 		!!xenbus_read_unsigned(info->xbdev->otherend, "discard-secure",
2170 				       0);
2171 }
2172 
2173 static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo)
2174 {
2175 	unsigned int psegs, grants;
2176 	int err, i;
2177 	struct blkfront_info *info = rinfo->dev_info;
2178 
2179 	if (info->max_indirect_segments == 0) {
2180 		if (!HAS_EXTRA_REQ)
2181 			grants = BLKIF_MAX_SEGMENTS_PER_REQUEST;
2182 		else {
2183 			/*
2184 			 * When an extra req is required, the maximum
2185 			 * grants supported is related to the size of the
2186 			 * Linux block segment.
2187 			 */
2188 			grants = GRANTS_PER_PSEG;
2189 		}
2190 	}
2191 	else
2192 		grants = info->max_indirect_segments;
2193 	psegs = DIV_ROUND_UP(grants, GRANTS_PER_PSEG);
2194 
2195 	err = fill_grant_buffer(rinfo,
2196 				(grants + INDIRECT_GREFS(grants)) * BLK_RING_SIZE(info));
2197 	if (err)
2198 		goto out_of_memory;
2199 
2200 	if (!info->feature_persistent && info->max_indirect_segments) {
2201 		/*
2202 		 * We are using indirect descriptors but not persistent
2203 		 * grants, we need to allocate a set of pages that can be
2204 		 * used for mapping indirect grefs
2205 		 */
2206 		int num = INDIRECT_GREFS(grants) * BLK_RING_SIZE(info);
2207 
2208 		BUG_ON(!list_empty(&rinfo->indirect_pages));
2209 		for (i = 0; i < num; i++) {
2210 			struct page *indirect_page = alloc_page(GFP_NOIO);
2211 			if (!indirect_page)
2212 				goto out_of_memory;
2213 			list_add(&indirect_page->lru, &rinfo->indirect_pages);
2214 		}
2215 	}
2216 
2217 	for (i = 0; i < BLK_RING_SIZE(info); i++) {
2218 		rinfo->shadow[i].grants_used = kzalloc(
2219 			sizeof(rinfo->shadow[i].grants_used[0]) * grants,
2220 			GFP_NOIO);
2221 		rinfo->shadow[i].sg = kzalloc(sizeof(rinfo->shadow[i].sg[0]) * psegs, GFP_NOIO);
2222 		if (info->max_indirect_segments)
2223 			rinfo->shadow[i].indirect_grants = kzalloc(
2224 				sizeof(rinfo->shadow[i].indirect_grants[0]) *
2225 				INDIRECT_GREFS(grants),
2226 				GFP_NOIO);
2227 		if ((rinfo->shadow[i].grants_used == NULL) ||
2228 			(rinfo->shadow[i].sg == NULL) ||
2229 		     (info->max_indirect_segments &&
2230 		     (rinfo->shadow[i].indirect_grants == NULL)))
2231 			goto out_of_memory;
2232 		sg_init_table(rinfo->shadow[i].sg, psegs);
2233 	}
2234 
2235 
2236 	return 0;
2237 
2238 out_of_memory:
2239 	for (i = 0; i < BLK_RING_SIZE(info); i++) {
2240 		kfree(rinfo->shadow[i].grants_used);
2241 		rinfo->shadow[i].grants_used = NULL;
2242 		kfree(rinfo->shadow[i].sg);
2243 		rinfo->shadow[i].sg = NULL;
2244 		kfree(rinfo->shadow[i].indirect_grants);
2245 		rinfo->shadow[i].indirect_grants = NULL;
2246 	}
2247 	if (!list_empty(&rinfo->indirect_pages)) {
2248 		struct page *indirect_page, *n;
2249 		list_for_each_entry_safe(indirect_page, n, &rinfo->indirect_pages, lru) {
2250 			list_del(&indirect_page->lru);
2251 			__free_page(indirect_page);
2252 		}
2253 	}
2254 	return -ENOMEM;
2255 }
2256 
2257 /*
2258  * Gather all backend feature-*
2259  */
2260 static void blkfront_gather_backend_features(struct blkfront_info *info)
2261 {
2262 	unsigned int indirect_segments;
2263 
2264 	info->feature_flush = 0;
2265 	info->feature_fua = 0;
2266 
2267 	/*
2268 	 * If there's no "feature-barrier" defined, then it means
2269 	 * we're dealing with a very old backend which writes
2270 	 * synchronously; nothing to do.
2271 	 *
2272 	 * If there are barriers, then we use flush.
2273 	 */
2274 	if (xenbus_read_unsigned(info->xbdev->otherend, "feature-barrier", 0)) {
2275 		info->feature_flush = 1;
2276 		info->feature_fua = 1;
2277 	}
2278 
2279 	/*
2280 	 * And if there is "feature-flush-cache" use that above
2281 	 * barriers.
2282 	 */
2283 	if (xenbus_read_unsigned(info->xbdev->otherend, "feature-flush-cache",
2284 				 0)) {
2285 		info->feature_flush = 1;
2286 		info->feature_fua = 0;
2287 	}
2288 
2289 	if (xenbus_read_unsigned(info->xbdev->otherend, "feature-discard", 0))
2290 		blkfront_setup_discard(info);
2291 
2292 	info->feature_persistent =
2293 		!!xenbus_read_unsigned(info->xbdev->otherend,
2294 				       "feature-persistent", 0);
2295 
2296 	indirect_segments = xenbus_read_unsigned(info->xbdev->otherend,
2297 					"feature-max-indirect-segments", 0);
2298 	if (indirect_segments > xen_blkif_max_segments)
2299 		indirect_segments = xen_blkif_max_segments;
2300 	if (indirect_segments <= BLKIF_MAX_SEGMENTS_PER_REQUEST)
2301 		indirect_segments = 0;
2302 	info->max_indirect_segments = indirect_segments;
2303 }
2304 
2305 /*
2306  * Invoked when the backend is finally 'ready' (and has told produced
2307  * the details about the physical device - #sectors, size, etc).
2308  */
2309 static void blkfront_connect(struct blkfront_info *info)
2310 {
2311 	unsigned long long sectors;
2312 	unsigned long sector_size;
2313 	unsigned int physical_sector_size;
2314 	unsigned int binfo;
2315 	char *envp[] = { "RESIZE=1", NULL };
2316 	int err, i;
2317 
2318 	switch (info->connected) {
2319 	case BLKIF_STATE_CONNECTED:
2320 		/*
2321 		 * Potentially, the back-end may be signalling
2322 		 * a capacity change; update the capacity.
2323 		 */
2324 		err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
2325 				   "sectors", "%Lu", &sectors);
2326 		if (XENBUS_EXIST_ERR(err))
2327 			return;
2328 		printk(KERN_INFO "Setting capacity to %Lu\n",
2329 		       sectors);
2330 		set_capacity(info->gd, sectors);
2331 		revalidate_disk(info->gd);
2332 		kobject_uevent_env(&disk_to_dev(info->gd)->kobj,
2333 				   KOBJ_CHANGE, envp);
2334 
2335 		return;
2336 	case BLKIF_STATE_SUSPENDED:
2337 		/*
2338 		 * If we are recovering from suspension, we need to wait
2339 		 * for the backend to announce it's features before
2340 		 * reconnecting, at least we need to know if the backend
2341 		 * supports indirect descriptors, and how many.
2342 		 */
2343 		blkif_recover(info);
2344 		return;
2345 
2346 	default:
2347 		break;
2348 	}
2349 
2350 	dev_dbg(&info->xbdev->dev, "%s:%s.\n",
2351 		__func__, info->xbdev->otherend);
2352 
2353 	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
2354 			    "sectors", "%llu", &sectors,
2355 			    "info", "%u", &binfo,
2356 			    "sector-size", "%lu", &sector_size,
2357 			    NULL);
2358 	if (err) {
2359 		xenbus_dev_fatal(info->xbdev, err,
2360 				 "reading backend fields at %s",
2361 				 info->xbdev->otherend);
2362 		return;
2363 	}
2364 
2365 	/*
2366 	 * physcial-sector-size is a newer field, so old backends may not
2367 	 * provide this. Assume physical sector size to be the same as
2368 	 * sector_size in that case.
2369 	 */
2370 	physical_sector_size = xenbus_read_unsigned(info->xbdev->otherend,
2371 						    "physical-sector-size",
2372 						    sector_size);
2373 	blkfront_gather_backend_features(info);
2374 	for (i = 0; i < info->nr_rings; i++) {
2375 		err = blkfront_setup_indirect(&info->rinfo[i]);
2376 		if (err) {
2377 			xenbus_dev_fatal(info->xbdev, err, "setup_indirect at %s",
2378 					 info->xbdev->otherend);
2379 			blkif_free(info, 0);
2380 			break;
2381 		}
2382 	}
2383 
2384 	err = xlvbd_alloc_gendisk(sectors, info, binfo, sector_size,
2385 				  physical_sector_size);
2386 	if (err) {
2387 		xenbus_dev_fatal(info->xbdev, err, "xlvbd_add at %s",
2388 				 info->xbdev->otherend);
2389 		goto fail;
2390 	}
2391 
2392 	xenbus_switch_state(info->xbdev, XenbusStateConnected);
2393 
2394 	/* Kick pending requests. */
2395 	info->connected = BLKIF_STATE_CONNECTED;
2396 	for (i = 0; i < info->nr_rings; i++)
2397 		kick_pending_request_queues(&info->rinfo[i]);
2398 
2399 	device_add_disk(&info->xbdev->dev, info->gd);
2400 
2401 	info->is_ready = 1;
2402 	return;
2403 
2404 fail:
2405 	blkif_free(info, 0);
2406 	return;
2407 }
2408 
2409 /**
2410  * Callback received when the backend's state changes.
2411  */
2412 static void blkback_changed(struct xenbus_device *dev,
2413 			    enum xenbus_state backend_state)
2414 {
2415 	struct blkfront_info *info = dev_get_drvdata(&dev->dev);
2416 
2417 	dev_dbg(&dev->dev, "blkfront:blkback_changed to state %d.\n", backend_state);
2418 
2419 	switch (backend_state) {
2420 	case XenbusStateInitWait:
2421 		if (dev->state != XenbusStateInitialising)
2422 			break;
2423 		if (talk_to_blkback(dev, info))
2424 			break;
2425 	case XenbusStateInitialising:
2426 	case XenbusStateInitialised:
2427 	case XenbusStateReconfiguring:
2428 	case XenbusStateReconfigured:
2429 	case XenbusStateUnknown:
2430 		break;
2431 
2432 	case XenbusStateConnected:
2433 		/*
2434 		 * talk_to_blkback sets state to XenbusStateInitialised
2435 		 * and blkfront_connect sets it to XenbusStateConnected
2436 		 * (if connection went OK).
2437 		 *
2438 		 * If the backend (or toolstack) decides to poke at backend
2439 		 * state (and re-trigger the watch by setting the state repeatedly
2440 		 * to XenbusStateConnected (4)) we need to deal with this.
2441 		 * This is allowed as this is used to communicate to the guest
2442 		 * that the size of disk has changed!
2443 		 */
2444 		if ((dev->state != XenbusStateInitialised) &&
2445 		    (dev->state != XenbusStateConnected)) {
2446 			if (talk_to_blkback(dev, info))
2447 				break;
2448 		}
2449 
2450 		blkfront_connect(info);
2451 		break;
2452 
2453 	case XenbusStateClosed:
2454 		if (dev->state == XenbusStateClosed)
2455 			break;
2456 		/* Missed the backend's Closing state -- fallthrough */
2457 	case XenbusStateClosing:
2458 		if (info)
2459 			blkfront_closing(info);
2460 		break;
2461 	}
2462 }
2463 
2464 static int blkfront_remove(struct xenbus_device *xbdev)
2465 {
2466 	struct blkfront_info *info = dev_get_drvdata(&xbdev->dev);
2467 	struct block_device *bdev = NULL;
2468 	struct gendisk *disk;
2469 
2470 	dev_dbg(&xbdev->dev, "%s removed", xbdev->nodename);
2471 
2472 	blkif_free(info, 0);
2473 
2474 	mutex_lock(&info->mutex);
2475 
2476 	disk = info->gd;
2477 	if (disk)
2478 		bdev = bdget_disk(disk, 0);
2479 
2480 	info->xbdev = NULL;
2481 	mutex_unlock(&info->mutex);
2482 
2483 	if (!bdev) {
2484 		kfree(info);
2485 		return 0;
2486 	}
2487 
2488 	/*
2489 	 * The xbdev was removed before we reached the Closed
2490 	 * state. See if it's safe to remove the disk. If the bdev
2491 	 * isn't closed yet, we let release take care of it.
2492 	 */
2493 
2494 	mutex_lock(&bdev->bd_mutex);
2495 	info = disk->private_data;
2496 
2497 	dev_warn(disk_to_dev(disk),
2498 		 "%s was hot-unplugged, %d stale handles\n",
2499 		 xbdev->nodename, bdev->bd_openers);
2500 
2501 	if (info && !bdev->bd_openers) {
2502 		xlvbd_release_gendisk(info);
2503 		disk->private_data = NULL;
2504 		kfree(info);
2505 	}
2506 
2507 	mutex_unlock(&bdev->bd_mutex);
2508 	bdput(bdev);
2509 
2510 	return 0;
2511 }
2512 
2513 static int blkfront_is_ready(struct xenbus_device *dev)
2514 {
2515 	struct blkfront_info *info = dev_get_drvdata(&dev->dev);
2516 
2517 	return info->is_ready && info->xbdev;
2518 }
2519 
2520 static int blkif_open(struct block_device *bdev, fmode_t mode)
2521 {
2522 	struct gendisk *disk = bdev->bd_disk;
2523 	struct blkfront_info *info;
2524 	int err = 0;
2525 
2526 	mutex_lock(&blkfront_mutex);
2527 
2528 	info = disk->private_data;
2529 	if (!info) {
2530 		/* xbdev gone */
2531 		err = -ERESTARTSYS;
2532 		goto out;
2533 	}
2534 
2535 	mutex_lock(&info->mutex);
2536 
2537 	if (!info->gd)
2538 		/* xbdev is closed */
2539 		err = -ERESTARTSYS;
2540 
2541 	mutex_unlock(&info->mutex);
2542 
2543 out:
2544 	mutex_unlock(&blkfront_mutex);
2545 	return err;
2546 }
2547 
2548 static void blkif_release(struct gendisk *disk, fmode_t mode)
2549 {
2550 	struct blkfront_info *info = disk->private_data;
2551 	struct block_device *bdev;
2552 	struct xenbus_device *xbdev;
2553 
2554 	mutex_lock(&blkfront_mutex);
2555 
2556 	bdev = bdget_disk(disk, 0);
2557 
2558 	if (!bdev) {
2559 		WARN(1, "Block device %s yanked out from us!\n", disk->disk_name);
2560 		goto out_mutex;
2561 	}
2562 	if (bdev->bd_openers)
2563 		goto out;
2564 
2565 	/*
2566 	 * Check if we have been instructed to close. We will have
2567 	 * deferred this request, because the bdev was still open.
2568 	 */
2569 
2570 	mutex_lock(&info->mutex);
2571 	xbdev = info->xbdev;
2572 
2573 	if (xbdev && xbdev->state == XenbusStateClosing) {
2574 		/* pending switch to state closed */
2575 		dev_info(disk_to_dev(bdev->bd_disk), "releasing disk\n");
2576 		xlvbd_release_gendisk(info);
2577 		xenbus_frontend_closed(info->xbdev);
2578  	}
2579 
2580 	mutex_unlock(&info->mutex);
2581 
2582 	if (!xbdev) {
2583 		/* sudden device removal */
2584 		dev_info(disk_to_dev(bdev->bd_disk), "releasing disk\n");
2585 		xlvbd_release_gendisk(info);
2586 		disk->private_data = NULL;
2587 		kfree(info);
2588 	}
2589 
2590 out:
2591 	bdput(bdev);
2592 out_mutex:
2593 	mutex_unlock(&blkfront_mutex);
2594 }
2595 
2596 static const struct block_device_operations xlvbd_block_fops =
2597 {
2598 	.owner = THIS_MODULE,
2599 	.open = blkif_open,
2600 	.release = blkif_release,
2601 	.getgeo = blkif_getgeo,
2602 	.ioctl = blkif_ioctl,
2603 };
2604 
2605 
2606 static const struct xenbus_device_id blkfront_ids[] = {
2607 	{ "vbd" },
2608 	{ "" }
2609 };
2610 
2611 static struct xenbus_driver blkfront_driver = {
2612 	.ids  = blkfront_ids,
2613 	.probe = blkfront_probe,
2614 	.remove = blkfront_remove,
2615 	.resume = blkfront_resume,
2616 	.otherend_changed = blkback_changed,
2617 	.is_ready = blkfront_is_ready,
2618 };
2619 
2620 static int __init xlblk_init(void)
2621 {
2622 	int ret;
2623 	int nr_cpus = num_online_cpus();
2624 
2625 	if (!xen_domain())
2626 		return -ENODEV;
2627 
2628 	if (xen_blkif_max_segments < BLKIF_MAX_SEGMENTS_PER_REQUEST)
2629 		xen_blkif_max_segments = BLKIF_MAX_SEGMENTS_PER_REQUEST;
2630 
2631 	if (xen_blkif_max_ring_order > XENBUS_MAX_RING_GRANT_ORDER) {
2632 		pr_info("Invalid max_ring_order (%d), will use default max: %d.\n",
2633 			xen_blkif_max_ring_order, XENBUS_MAX_RING_GRANT_ORDER);
2634 		xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER;
2635 	}
2636 
2637 	if (xen_blkif_max_queues > nr_cpus) {
2638 		pr_info("Invalid max_queues (%d), will use default max: %d.\n",
2639 			xen_blkif_max_queues, nr_cpus);
2640 		xen_blkif_max_queues = nr_cpus;
2641 	}
2642 
2643 	if (!xen_has_pv_disk_devices())
2644 		return -ENODEV;
2645 
2646 	if (register_blkdev(XENVBD_MAJOR, DEV_NAME)) {
2647 		printk(KERN_WARNING "xen_blk: can't get major %d with name %s\n",
2648 		       XENVBD_MAJOR, DEV_NAME);
2649 		return -ENODEV;
2650 	}
2651 
2652 	ret = xenbus_register_frontend(&blkfront_driver);
2653 	if (ret) {
2654 		unregister_blkdev(XENVBD_MAJOR, DEV_NAME);
2655 		return ret;
2656 	}
2657 
2658 	return 0;
2659 }
2660 module_init(xlblk_init);
2661 
2662 
2663 static void __exit xlblk_exit(void)
2664 {
2665 	xenbus_unregister_driver(&blkfront_driver);
2666 	unregister_blkdev(XENVBD_MAJOR, DEV_NAME);
2667 	kfree(minors);
2668 }
2669 module_exit(xlblk_exit);
2670 
2671 MODULE_DESCRIPTION("Xen virtual block device frontend");
2672 MODULE_LICENSE("GPL");
2673 MODULE_ALIAS_BLOCKDEV_MAJOR(XENVBD_MAJOR);
2674 MODULE_ALIAS("xen:vbd");
2675 MODULE_ALIAS("xenblk");
2676