1 /****************************************************************************** 2 * 3 * Back-end of the driver for virtual block devices. This portion of the 4 * driver exports a 'unified' block-device interface that can be accessed 5 * by any operating system that implements a compatible front end. A 6 * reference front-end implementation can be found in: 7 * drivers/block/xen-blkfront.c 8 * 9 * Copyright (c) 2003-2004, Keir Fraser & Steve Hand 10 * Copyright (c) 2005, Christopher Clark 11 * 12 * This program is free software; you can redistribute it and/or 13 * modify it under the terms of the GNU General Public License version 2 14 * as published by the Free Software Foundation; or, when distributed 15 * separately from the Linux kernel or incorporated into other 16 * software packages, subject to the following license: 17 * 18 * Permission is hereby granted, free of charge, to any person obtaining a copy 19 * of this source file (the "Software"), to deal in the Software without 20 * restriction, including without limitation the rights to use, copy, modify, 21 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 22 * and to permit persons to whom the Software is furnished to do so, subject to 23 * the following conditions: 24 * 25 * The above copyright notice and this permission notice shall be included in 26 * all copies or substantial portions of the Software. 27 * 28 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 29 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 30 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 31 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 32 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 33 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 34 * IN THE SOFTWARE. 35 */ 36 37 #define pr_fmt(fmt) "xen-blkback: " fmt 38 39 #include <linux/spinlock.h> 40 #include <linux/kthread.h> 41 #include <linux/list.h> 42 #include <linux/delay.h> 43 #include <linux/freezer.h> 44 #include <linux/bitmap.h> 45 46 #include <xen/events.h> 47 #include <xen/page.h> 48 #include <xen/xen.h> 49 #include <asm/xen/hypervisor.h> 50 #include <asm/xen/hypercall.h> 51 #include <xen/balloon.h> 52 #include <xen/grant_table.h> 53 #include "common.h" 54 55 /* 56 * Maximum number of unused free pages to keep in the internal buffer. 57 * Setting this to a value too low will reduce memory used in each backend, 58 * but can have a performance penalty. 59 * 60 * A sane value is xen_blkif_reqs * BLKIF_MAX_SEGMENTS_PER_REQUEST, but can 61 * be set to a lower value that might degrade performance on some intensive 62 * IO workloads. 63 */ 64 65 static int max_buffer_pages = 1024; 66 module_param_named(max_buffer_pages, max_buffer_pages, int, 0644); 67 MODULE_PARM_DESC(max_buffer_pages, 68 "Maximum number of free pages to keep in each block backend buffer"); 69 70 /* 71 * Maximum number of grants to map persistently in blkback. For maximum 72 * performance this should be the total numbers of grants that can be used 73 * to fill the ring, but since this might become too high, specially with 74 * the use of indirect descriptors, we set it to a value that provides good 75 * performance without using too much memory. 76 * 77 * When the list of persistent grants is full we clean it up using a LRU 78 * algorithm. 79 */ 80 81 static int max_pgrants = 1056; 82 module_param_named(max_persistent_grants, max_pgrants, int, 0644); 83 MODULE_PARM_DESC(max_persistent_grants, 84 "Maximum number of grants to map persistently"); 85 86 /* 87 * How long a persistent grant is allowed to remain allocated without being in 88 * use. The time is in seconds, 0 means indefinitely long. 89 */ 90 91 static unsigned int pgrant_timeout = 60; 92 module_param_named(persistent_grant_unused_seconds, pgrant_timeout, 93 uint, 0644); 94 MODULE_PARM_DESC(persistent_grant_unused_seconds, 95 "Time in seconds an unused persistent grant is allowed to " 96 "remain allocated. Default is 60, 0 means unlimited."); 97 98 /* 99 * Maximum number of rings/queues blkback supports, allow as many queues as there 100 * are CPUs if user has not specified a value. 101 */ 102 unsigned int xenblk_max_queues; 103 module_param_named(max_queues, xenblk_max_queues, uint, 0644); 104 MODULE_PARM_DESC(max_queues, 105 "Maximum number of hardware queues per virtual disk." \ 106 "By default it is the number of online CPUs."); 107 108 /* 109 * Maximum order of pages to be used for the shared ring between front and 110 * backend, 4KB page granularity is used. 111 */ 112 unsigned int xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER; 113 module_param_named(max_ring_page_order, xen_blkif_max_ring_order, int, 0444); 114 MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the shared ring"); 115 /* 116 * The LRU mechanism to clean the lists of persistent grants needs to 117 * be executed periodically. The time interval between consecutive executions 118 * of the purge mechanism is set in ms. 119 */ 120 #define LRU_INTERVAL 100 121 122 /* 123 * When the persistent grants list is full we will remove unused grants 124 * from the list. The percent number of grants to be removed at each LRU 125 * execution. 126 */ 127 #define LRU_PERCENT_CLEAN 5 128 129 /* Run-time switchable: /sys/module/blkback/parameters/ */ 130 static unsigned int log_stats; 131 module_param(log_stats, int, 0644); 132 133 #define BLKBACK_INVALID_HANDLE (~0) 134 135 static inline bool persistent_gnt_timeout(struct persistent_gnt *persistent_gnt) 136 { 137 return pgrant_timeout && (jiffies - persistent_gnt->last_used >= 138 HZ * pgrant_timeout); 139 } 140 141 #define vaddr(page) ((unsigned long)pfn_to_kaddr(page_to_pfn(page))) 142 143 static int do_block_io_op(struct xen_blkif_ring *ring, unsigned int *eoi_flags); 144 static int dispatch_rw_block_io(struct xen_blkif_ring *ring, 145 struct blkif_request *req, 146 struct pending_req *pending_req); 147 static void make_response(struct xen_blkif_ring *ring, u64 id, 148 unsigned short op, int st); 149 150 #define foreach_grant_safe(pos, n, rbtree, node) \ 151 for ((pos) = container_of(rb_first((rbtree)), typeof(*(pos)), node), \ 152 (n) = (&(pos)->node != NULL) ? rb_next(&(pos)->node) : NULL; \ 153 &(pos)->node != NULL; \ 154 (pos) = container_of(n, typeof(*(pos)), node), \ 155 (n) = (&(pos)->node != NULL) ? rb_next(&(pos)->node) : NULL) 156 157 158 /* 159 * We don't need locking around the persistent grant helpers 160 * because blkback uses a single-thread for each backend, so we 161 * can be sure that this functions will never be called recursively. 162 * 163 * The only exception to that is put_persistent_grant, that can be called 164 * from interrupt context (by xen_blkbk_unmap), so we have to use atomic 165 * bit operations to modify the flags of a persistent grant and to count 166 * the number of used grants. 167 */ 168 static int add_persistent_gnt(struct xen_blkif_ring *ring, 169 struct persistent_gnt *persistent_gnt) 170 { 171 struct rb_node **new = NULL, *parent = NULL; 172 struct persistent_gnt *this; 173 struct xen_blkif *blkif = ring->blkif; 174 175 if (ring->persistent_gnt_c >= max_pgrants) { 176 if (!blkif->vbd.overflow_max_grants) 177 blkif->vbd.overflow_max_grants = 1; 178 return -EBUSY; 179 } 180 /* Figure out where to put new node */ 181 new = &ring->persistent_gnts.rb_node; 182 while (*new) { 183 this = container_of(*new, struct persistent_gnt, node); 184 185 parent = *new; 186 if (persistent_gnt->gnt < this->gnt) 187 new = &((*new)->rb_left); 188 else if (persistent_gnt->gnt > this->gnt) 189 new = &((*new)->rb_right); 190 else { 191 pr_alert_ratelimited("trying to add a gref that's already in the tree\n"); 192 return -EINVAL; 193 } 194 } 195 196 persistent_gnt->active = true; 197 /* Add new node and rebalance tree. */ 198 rb_link_node(&(persistent_gnt->node), parent, new); 199 rb_insert_color(&(persistent_gnt->node), &ring->persistent_gnts); 200 ring->persistent_gnt_c++; 201 atomic_inc(&ring->persistent_gnt_in_use); 202 return 0; 203 } 204 205 static struct persistent_gnt *get_persistent_gnt(struct xen_blkif_ring *ring, 206 grant_ref_t gref) 207 { 208 struct persistent_gnt *data; 209 struct rb_node *node = NULL; 210 211 node = ring->persistent_gnts.rb_node; 212 while (node) { 213 data = container_of(node, struct persistent_gnt, node); 214 215 if (gref < data->gnt) 216 node = node->rb_left; 217 else if (gref > data->gnt) 218 node = node->rb_right; 219 else { 220 if (data->active) { 221 pr_alert_ratelimited("requesting a grant already in use\n"); 222 return NULL; 223 } 224 data->active = true; 225 atomic_inc(&ring->persistent_gnt_in_use); 226 return data; 227 } 228 } 229 return NULL; 230 } 231 232 static void put_persistent_gnt(struct xen_blkif_ring *ring, 233 struct persistent_gnt *persistent_gnt) 234 { 235 if (!persistent_gnt->active) 236 pr_alert_ratelimited("freeing a grant already unused\n"); 237 persistent_gnt->last_used = jiffies; 238 persistent_gnt->active = false; 239 atomic_dec(&ring->persistent_gnt_in_use); 240 } 241 242 static void free_persistent_gnts(struct xen_blkif_ring *ring, struct rb_root *root, 243 unsigned int num) 244 { 245 struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST]; 246 struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST]; 247 struct persistent_gnt *persistent_gnt; 248 struct rb_node *n; 249 int segs_to_unmap = 0; 250 struct gntab_unmap_queue_data unmap_data; 251 252 unmap_data.pages = pages; 253 unmap_data.unmap_ops = unmap; 254 unmap_data.kunmap_ops = NULL; 255 256 foreach_grant_safe(persistent_gnt, n, root, node) { 257 BUG_ON(persistent_gnt->handle == 258 BLKBACK_INVALID_HANDLE); 259 gnttab_set_unmap_op(&unmap[segs_to_unmap], 260 (unsigned long) pfn_to_kaddr(page_to_pfn( 261 persistent_gnt->page)), 262 GNTMAP_host_map, 263 persistent_gnt->handle); 264 265 pages[segs_to_unmap] = persistent_gnt->page; 266 267 if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST || 268 !rb_next(&persistent_gnt->node)) { 269 270 unmap_data.count = segs_to_unmap; 271 BUG_ON(gnttab_unmap_refs_sync(&unmap_data)); 272 273 gnttab_page_cache_put(&ring->free_pages, pages, 274 segs_to_unmap); 275 segs_to_unmap = 0; 276 } 277 278 rb_erase(&persistent_gnt->node, root); 279 kfree(persistent_gnt); 280 num--; 281 } 282 BUG_ON(num != 0); 283 } 284 285 void xen_blkbk_unmap_purged_grants(struct work_struct *work) 286 { 287 struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST]; 288 struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST]; 289 struct persistent_gnt *persistent_gnt; 290 int segs_to_unmap = 0; 291 struct xen_blkif_ring *ring = container_of(work, typeof(*ring), persistent_purge_work); 292 struct gntab_unmap_queue_data unmap_data; 293 294 unmap_data.pages = pages; 295 unmap_data.unmap_ops = unmap; 296 unmap_data.kunmap_ops = NULL; 297 298 while(!list_empty(&ring->persistent_purge_list)) { 299 persistent_gnt = list_first_entry(&ring->persistent_purge_list, 300 struct persistent_gnt, 301 remove_node); 302 list_del(&persistent_gnt->remove_node); 303 304 gnttab_set_unmap_op(&unmap[segs_to_unmap], 305 vaddr(persistent_gnt->page), 306 GNTMAP_host_map, 307 persistent_gnt->handle); 308 309 pages[segs_to_unmap] = persistent_gnt->page; 310 311 if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST) { 312 unmap_data.count = segs_to_unmap; 313 BUG_ON(gnttab_unmap_refs_sync(&unmap_data)); 314 gnttab_page_cache_put(&ring->free_pages, pages, 315 segs_to_unmap); 316 segs_to_unmap = 0; 317 } 318 kfree(persistent_gnt); 319 } 320 if (segs_to_unmap > 0) { 321 unmap_data.count = segs_to_unmap; 322 BUG_ON(gnttab_unmap_refs_sync(&unmap_data)); 323 gnttab_page_cache_put(&ring->free_pages, pages, segs_to_unmap); 324 } 325 } 326 327 static void purge_persistent_gnt(struct xen_blkif_ring *ring) 328 { 329 struct persistent_gnt *persistent_gnt; 330 struct rb_node *n; 331 unsigned int num_clean, total; 332 bool scan_used = false; 333 struct rb_root *root; 334 335 if (work_busy(&ring->persistent_purge_work)) { 336 pr_alert_ratelimited("Scheduled work from previous purge is still busy, cannot purge list\n"); 337 goto out; 338 } 339 340 if (ring->persistent_gnt_c < max_pgrants || 341 (ring->persistent_gnt_c == max_pgrants && 342 !ring->blkif->vbd.overflow_max_grants)) { 343 num_clean = 0; 344 } else { 345 num_clean = (max_pgrants / 100) * LRU_PERCENT_CLEAN; 346 num_clean = ring->persistent_gnt_c - max_pgrants + num_clean; 347 num_clean = min(ring->persistent_gnt_c, num_clean); 348 pr_debug("Going to purge at least %u persistent grants\n", 349 num_clean); 350 } 351 352 /* 353 * At this point, we can assure that there will be no calls 354 * to get_persistent_grant (because we are executing this code from 355 * xen_blkif_schedule), there can only be calls to put_persistent_gnt, 356 * which means that the number of currently used grants will go down, 357 * but never up, so we will always be able to remove the requested 358 * number of grants. 359 */ 360 361 total = 0; 362 363 BUG_ON(!list_empty(&ring->persistent_purge_list)); 364 root = &ring->persistent_gnts; 365 purge_list: 366 foreach_grant_safe(persistent_gnt, n, root, node) { 367 BUG_ON(persistent_gnt->handle == 368 BLKBACK_INVALID_HANDLE); 369 370 if (persistent_gnt->active) 371 continue; 372 if (!scan_used && !persistent_gnt_timeout(persistent_gnt)) 373 continue; 374 if (scan_used && total >= num_clean) 375 continue; 376 377 rb_erase(&persistent_gnt->node, root); 378 list_add(&persistent_gnt->remove_node, 379 &ring->persistent_purge_list); 380 total++; 381 } 382 /* 383 * Check whether we also need to start cleaning 384 * grants that were used since last purge in order to cope 385 * with the requested num 386 */ 387 if (!scan_used && total < num_clean) { 388 pr_debug("Still missing %u purged frames\n", num_clean - total); 389 scan_used = true; 390 goto purge_list; 391 } 392 393 if (total) { 394 ring->persistent_gnt_c -= total; 395 ring->blkif->vbd.overflow_max_grants = 0; 396 397 /* We can defer this work */ 398 schedule_work(&ring->persistent_purge_work); 399 pr_debug("Purged %u/%u\n", num_clean, total); 400 } 401 402 out: 403 return; 404 } 405 406 /* 407 * Retrieve from the 'pending_reqs' a free pending_req structure to be used. 408 */ 409 static struct pending_req *alloc_req(struct xen_blkif_ring *ring) 410 { 411 struct pending_req *req = NULL; 412 unsigned long flags; 413 414 spin_lock_irqsave(&ring->pending_free_lock, flags); 415 if (!list_empty(&ring->pending_free)) { 416 req = list_entry(ring->pending_free.next, struct pending_req, 417 free_list); 418 list_del(&req->free_list); 419 } 420 spin_unlock_irqrestore(&ring->pending_free_lock, flags); 421 return req; 422 } 423 424 /* 425 * Return the 'pending_req' structure back to the freepool. We also 426 * wake up the thread if it was waiting for a free page. 427 */ 428 static void free_req(struct xen_blkif_ring *ring, struct pending_req *req) 429 { 430 unsigned long flags; 431 int was_empty; 432 433 spin_lock_irqsave(&ring->pending_free_lock, flags); 434 was_empty = list_empty(&ring->pending_free); 435 list_add(&req->free_list, &ring->pending_free); 436 spin_unlock_irqrestore(&ring->pending_free_lock, flags); 437 if (was_empty) 438 wake_up(&ring->pending_free_wq); 439 } 440 441 /* 442 * Routines for managing virtual block devices (vbds). 443 */ 444 static int xen_vbd_translate(struct phys_req *req, struct xen_blkif *blkif, 445 int operation) 446 { 447 struct xen_vbd *vbd = &blkif->vbd; 448 int rc = -EACCES; 449 450 if ((operation != REQ_OP_READ) && vbd->readonly) 451 goto out; 452 453 if (likely(req->nr_sects)) { 454 blkif_sector_t end = req->sector_number + req->nr_sects; 455 456 if (unlikely(end < req->sector_number)) 457 goto out; 458 if (unlikely(end > vbd_sz(vbd))) 459 goto out; 460 } 461 462 req->dev = vbd->pdevice; 463 req->bdev = vbd->bdev; 464 rc = 0; 465 466 out: 467 return rc; 468 } 469 470 static void xen_vbd_resize(struct xen_blkif *blkif) 471 { 472 struct xen_vbd *vbd = &blkif->vbd; 473 struct xenbus_transaction xbt; 474 int err; 475 struct xenbus_device *dev = xen_blkbk_xenbus(blkif->be); 476 unsigned long long new_size = vbd_sz(vbd); 477 478 pr_info("VBD Resize: Domid: %d, Device: (%d, %d)\n", 479 blkif->domid, MAJOR(vbd->pdevice), MINOR(vbd->pdevice)); 480 pr_info("VBD Resize: new size %llu\n", new_size); 481 vbd->size = new_size; 482 again: 483 err = xenbus_transaction_start(&xbt); 484 if (err) { 485 pr_warn("Error starting transaction\n"); 486 return; 487 } 488 err = xenbus_printf(xbt, dev->nodename, "sectors", "%llu", 489 (unsigned long long)vbd_sz(vbd)); 490 if (err) { 491 pr_warn("Error writing new size\n"); 492 goto abort; 493 } 494 /* 495 * Write the current state; we will use this to synchronize 496 * the front-end. If the current state is "connected" the 497 * front-end will get the new size information online. 498 */ 499 err = xenbus_printf(xbt, dev->nodename, "state", "%d", dev->state); 500 if (err) { 501 pr_warn("Error writing the state\n"); 502 goto abort; 503 } 504 505 err = xenbus_transaction_end(xbt, 0); 506 if (err == -EAGAIN) 507 goto again; 508 if (err) 509 pr_warn("Error ending transaction\n"); 510 return; 511 abort: 512 xenbus_transaction_end(xbt, 1); 513 } 514 515 /* 516 * Notification from the guest OS. 517 */ 518 static void blkif_notify_work(struct xen_blkif_ring *ring) 519 { 520 ring->waiting_reqs = 1; 521 wake_up(&ring->wq); 522 } 523 524 irqreturn_t xen_blkif_be_int(int irq, void *dev_id) 525 { 526 blkif_notify_work(dev_id); 527 return IRQ_HANDLED; 528 } 529 530 /* 531 * SCHEDULER FUNCTIONS 532 */ 533 534 static void print_stats(struct xen_blkif_ring *ring) 535 { 536 pr_info("(%s): oo %3llu | rd %4llu | wr %4llu | f %4llu" 537 " | ds %4llu | pg: %4u/%4d\n", 538 current->comm, ring->st_oo_req, 539 ring->st_rd_req, ring->st_wr_req, 540 ring->st_f_req, ring->st_ds_req, 541 ring->persistent_gnt_c, max_pgrants); 542 ring->st_print = jiffies + msecs_to_jiffies(10 * 1000); 543 ring->st_rd_req = 0; 544 ring->st_wr_req = 0; 545 ring->st_oo_req = 0; 546 ring->st_ds_req = 0; 547 } 548 549 int xen_blkif_schedule(void *arg) 550 { 551 struct xen_blkif_ring *ring = arg; 552 struct xen_blkif *blkif = ring->blkif; 553 struct xen_vbd *vbd = &blkif->vbd; 554 unsigned long timeout; 555 int ret; 556 bool do_eoi; 557 unsigned int eoi_flags = XEN_EOI_FLAG_SPURIOUS; 558 559 set_freezable(); 560 while (!kthread_should_stop()) { 561 if (try_to_freeze()) 562 continue; 563 if (unlikely(vbd->size != vbd_sz(vbd))) 564 xen_vbd_resize(blkif); 565 566 timeout = msecs_to_jiffies(LRU_INTERVAL); 567 568 timeout = wait_event_interruptible_timeout( 569 ring->wq, 570 ring->waiting_reqs || kthread_should_stop(), 571 timeout); 572 if (timeout == 0) 573 goto purge_gnt_list; 574 timeout = wait_event_interruptible_timeout( 575 ring->pending_free_wq, 576 !list_empty(&ring->pending_free) || 577 kthread_should_stop(), 578 timeout); 579 if (timeout == 0) 580 goto purge_gnt_list; 581 582 do_eoi = ring->waiting_reqs; 583 584 ring->waiting_reqs = 0; 585 smp_mb(); /* clear flag *before* checking for work */ 586 587 ret = do_block_io_op(ring, &eoi_flags); 588 if (ret > 0) 589 ring->waiting_reqs = 1; 590 if (ret == -EACCES) 591 wait_event_interruptible(ring->shutdown_wq, 592 kthread_should_stop()); 593 594 if (do_eoi && !ring->waiting_reqs) { 595 xen_irq_lateeoi(ring->irq, eoi_flags); 596 eoi_flags |= XEN_EOI_FLAG_SPURIOUS; 597 } 598 599 purge_gnt_list: 600 if (blkif->vbd.feature_gnt_persistent && 601 time_after(jiffies, ring->next_lru)) { 602 purge_persistent_gnt(ring); 603 ring->next_lru = jiffies + msecs_to_jiffies(LRU_INTERVAL); 604 } 605 606 /* Shrink the free pages pool if it is too large. */ 607 if (time_before(jiffies, blkif->buffer_squeeze_end)) 608 gnttab_page_cache_shrink(&ring->free_pages, 0); 609 else 610 gnttab_page_cache_shrink(&ring->free_pages, 611 max_buffer_pages); 612 613 if (log_stats && time_after(jiffies, ring->st_print)) 614 print_stats(ring); 615 } 616 617 /* Drain pending purge work */ 618 flush_work(&ring->persistent_purge_work); 619 620 if (log_stats) 621 print_stats(ring); 622 623 ring->xenblkd = NULL; 624 625 return 0; 626 } 627 628 /* 629 * Remove persistent grants and empty the pool of free pages 630 */ 631 void xen_blkbk_free_caches(struct xen_blkif_ring *ring) 632 { 633 /* Free all persistent grant pages */ 634 if (!RB_EMPTY_ROOT(&ring->persistent_gnts)) 635 free_persistent_gnts(ring, &ring->persistent_gnts, 636 ring->persistent_gnt_c); 637 638 BUG_ON(!RB_EMPTY_ROOT(&ring->persistent_gnts)); 639 ring->persistent_gnt_c = 0; 640 641 /* Since we are shutting down remove all pages from the buffer */ 642 gnttab_page_cache_shrink(&ring->free_pages, 0 /* All */); 643 } 644 645 static unsigned int xen_blkbk_unmap_prepare( 646 struct xen_blkif_ring *ring, 647 struct grant_page **pages, 648 unsigned int num, 649 struct gnttab_unmap_grant_ref *unmap_ops, 650 struct page **unmap_pages) 651 { 652 unsigned int i, invcount = 0; 653 654 for (i = 0; i < num; i++) { 655 if (pages[i]->persistent_gnt != NULL) { 656 put_persistent_gnt(ring, pages[i]->persistent_gnt); 657 continue; 658 } 659 if (pages[i]->handle == BLKBACK_INVALID_HANDLE) 660 continue; 661 unmap_pages[invcount] = pages[i]->page; 662 gnttab_set_unmap_op(&unmap_ops[invcount], vaddr(pages[i]->page), 663 GNTMAP_host_map, pages[i]->handle); 664 pages[i]->handle = BLKBACK_INVALID_HANDLE; 665 invcount++; 666 } 667 668 return invcount; 669 } 670 671 static void xen_blkbk_unmap_and_respond_callback(int result, struct gntab_unmap_queue_data *data) 672 { 673 struct pending_req *pending_req = (struct pending_req *)(data->data); 674 struct xen_blkif_ring *ring = pending_req->ring; 675 struct xen_blkif *blkif = ring->blkif; 676 677 /* BUG_ON used to reproduce existing behaviour, 678 but is this the best way to deal with this? */ 679 BUG_ON(result); 680 681 gnttab_page_cache_put(&ring->free_pages, data->pages, data->count); 682 make_response(ring, pending_req->id, 683 pending_req->operation, pending_req->status); 684 free_req(ring, pending_req); 685 /* 686 * Make sure the request is freed before releasing blkif, 687 * or there could be a race between free_req and the 688 * cleanup done in xen_blkif_free during shutdown. 689 * 690 * NB: The fact that we might try to wake up pending_free_wq 691 * before drain_complete (in case there's a drain going on) 692 * it's not a problem with our current implementation 693 * because we can assure there's no thread waiting on 694 * pending_free_wq if there's a drain going on, but it has 695 * to be taken into account if the current model is changed. 696 */ 697 if (atomic_dec_and_test(&ring->inflight) && atomic_read(&blkif->drain)) { 698 complete(&blkif->drain_complete); 699 } 700 xen_blkif_put(blkif); 701 } 702 703 static void xen_blkbk_unmap_and_respond(struct pending_req *req) 704 { 705 struct gntab_unmap_queue_data* work = &req->gnttab_unmap_data; 706 struct xen_blkif_ring *ring = req->ring; 707 struct grant_page **pages = req->segments; 708 unsigned int invcount; 709 710 invcount = xen_blkbk_unmap_prepare(ring, pages, req->nr_segs, 711 req->unmap, req->unmap_pages); 712 713 work->data = req; 714 work->done = xen_blkbk_unmap_and_respond_callback; 715 work->unmap_ops = req->unmap; 716 work->kunmap_ops = NULL; 717 work->pages = req->unmap_pages; 718 work->count = invcount; 719 720 gnttab_unmap_refs_async(&req->gnttab_unmap_data); 721 } 722 723 724 /* 725 * Unmap the grant references. 726 * 727 * This could accumulate ops up to the batch size to reduce the number 728 * of hypercalls, but since this is only used in error paths there's 729 * no real need. 730 */ 731 static void xen_blkbk_unmap(struct xen_blkif_ring *ring, 732 struct grant_page *pages[], 733 int num) 734 { 735 struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST]; 736 struct page *unmap_pages[BLKIF_MAX_SEGMENTS_PER_REQUEST]; 737 unsigned int invcount = 0; 738 int ret; 739 740 while (num) { 741 unsigned int batch = min(num, BLKIF_MAX_SEGMENTS_PER_REQUEST); 742 743 invcount = xen_blkbk_unmap_prepare(ring, pages, batch, 744 unmap, unmap_pages); 745 if (invcount) { 746 ret = gnttab_unmap_refs(unmap, NULL, unmap_pages, invcount); 747 BUG_ON(ret); 748 gnttab_page_cache_put(&ring->free_pages, unmap_pages, 749 invcount); 750 } 751 pages += batch; 752 num -= batch; 753 } 754 } 755 756 static int xen_blkbk_map(struct xen_blkif_ring *ring, 757 struct grant_page *pages[], 758 int num, bool ro) 759 { 760 struct gnttab_map_grant_ref map[BLKIF_MAX_SEGMENTS_PER_REQUEST]; 761 struct page *pages_to_gnt[BLKIF_MAX_SEGMENTS_PER_REQUEST]; 762 struct persistent_gnt *persistent_gnt = NULL; 763 phys_addr_t addr = 0; 764 int i, seg_idx, new_map_idx; 765 int segs_to_map = 0; 766 int ret = 0; 767 int last_map = 0, map_until = 0; 768 int use_persistent_gnts; 769 struct xen_blkif *blkif = ring->blkif; 770 771 use_persistent_gnts = (blkif->vbd.feature_gnt_persistent); 772 773 /* 774 * Fill out preq.nr_sects with proper amount of sectors, and setup 775 * assign map[..] with the PFN of the page in our domain with the 776 * corresponding grant reference for each page. 777 */ 778 again: 779 for (i = map_until; i < num; i++) { 780 uint32_t flags; 781 782 if (use_persistent_gnts) { 783 persistent_gnt = get_persistent_gnt( 784 ring, 785 pages[i]->gref); 786 } 787 788 if (persistent_gnt) { 789 /* 790 * We are using persistent grants and 791 * the grant is already mapped 792 */ 793 pages[i]->page = persistent_gnt->page; 794 pages[i]->persistent_gnt = persistent_gnt; 795 } else { 796 if (gnttab_page_cache_get(&ring->free_pages, 797 &pages[i]->page)) 798 goto out_of_memory; 799 addr = vaddr(pages[i]->page); 800 pages_to_gnt[segs_to_map] = pages[i]->page; 801 pages[i]->persistent_gnt = NULL; 802 flags = GNTMAP_host_map; 803 if (!use_persistent_gnts && ro) 804 flags |= GNTMAP_readonly; 805 gnttab_set_map_op(&map[segs_to_map++], addr, 806 flags, pages[i]->gref, 807 blkif->domid); 808 } 809 map_until = i + 1; 810 if (segs_to_map == BLKIF_MAX_SEGMENTS_PER_REQUEST) 811 break; 812 } 813 814 if (segs_to_map) { 815 ret = gnttab_map_refs(map, NULL, pages_to_gnt, segs_to_map); 816 BUG_ON(ret); 817 } 818 819 /* 820 * Now swizzle the MFN in our domain with the MFN from the other domain 821 * so that when we access vaddr(pending_req,i) it has the contents of 822 * the page from the other domain. 823 */ 824 for (seg_idx = last_map, new_map_idx = 0; seg_idx < map_until; seg_idx++) { 825 if (!pages[seg_idx]->persistent_gnt) { 826 /* This is a newly mapped grant */ 827 BUG_ON(new_map_idx >= segs_to_map); 828 if (unlikely(map[new_map_idx].status != 0)) { 829 pr_debug("invalid buffer -- could not remap it\n"); 830 gnttab_page_cache_put(&ring->free_pages, 831 &pages[seg_idx]->page, 1); 832 pages[seg_idx]->handle = BLKBACK_INVALID_HANDLE; 833 ret |= 1; 834 goto next; 835 } 836 pages[seg_idx]->handle = map[new_map_idx].handle; 837 } else { 838 continue; 839 } 840 if (use_persistent_gnts && 841 ring->persistent_gnt_c < max_pgrants) { 842 /* 843 * We are using persistent grants, the grant is 844 * not mapped but we might have room for it. 845 */ 846 persistent_gnt = kmalloc(sizeof(struct persistent_gnt), 847 GFP_KERNEL); 848 if (!persistent_gnt) { 849 /* 850 * If we don't have enough memory to 851 * allocate the persistent_gnt struct 852 * map this grant non-persistenly 853 */ 854 goto next; 855 } 856 persistent_gnt->gnt = map[new_map_idx].ref; 857 persistent_gnt->handle = map[new_map_idx].handle; 858 persistent_gnt->page = pages[seg_idx]->page; 859 if (add_persistent_gnt(ring, 860 persistent_gnt)) { 861 kfree(persistent_gnt); 862 persistent_gnt = NULL; 863 goto next; 864 } 865 pages[seg_idx]->persistent_gnt = persistent_gnt; 866 pr_debug("grant %u added to the tree of persistent grants, using %u/%u\n", 867 persistent_gnt->gnt, ring->persistent_gnt_c, 868 max_pgrants); 869 goto next; 870 } 871 if (use_persistent_gnts && !blkif->vbd.overflow_max_grants) { 872 blkif->vbd.overflow_max_grants = 1; 873 pr_debug("domain %u, device %#x is using maximum number of persistent grants\n", 874 blkif->domid, blkif->vbd.handle); 875 } 876 /* 877 * We could not map this grant persistently, so use it as 878 * a non-persistent grant. 879 */ 880 next: 881 new_map_idx++; 882 } 883 segs_to_map = 0; 884 last_map = map_until; 885 if (map_until != num) 886 goto again; 887 888 return ret; 889 890 out_of_memory: 891 pr_alert("%s: out of memory\n", __func__); 892 gnttab_page_cache_put(&ring->free_pages, pages_to_gnt, segs_to_map); 893 for (i = last_map; i < num; i++) 894 pages[i]->handle = BLKBACK_INVALID_HANDLE; 895 return -ENOMEM; 896 } 897 898 static int xen_blkbk_map_seg(struct pending_req *pending_req) 899 { 900 int rc; 901 902 rc = xen_blkbk_map(pending_req->ring, pending_req->segments, 903 pending_req->nr_segs, 904 (pending_req->operation != BLKIF_OP_READ)); 905 906 return rc; 907 } 908 909 static int xen_blkbk_parse_indirect(struct blkif_request *req, 910 struct pending_req *pending_req, 911 struct seg_buf seg[], 912 struct phys_req *preq) 913 { 914 struct grant_page **pages = pending_req->indirect_pages; 915 struct xen_blkif_ring *ring = pending_req->ring; 916 int indirect_grefs, rc, n, nseg, i; 917 struct blkif_request_segment *segments = NULL; 918 919 nseg = pending_req->nr_segs; 920 indirect_grefs = INDIRECT_PAGES(nseg); 921 BUG_ON(indirect_grefs > BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST); 922 923 for (i = 0; i < indirect_grefs; i++) 924 pages[i]->gref = req->u.indirect.indirect_grefs[i]; 925 926 rc = xen_blkbk_map(ring, pages, indirect_grefs, true); 927 if (rc) 928 goto unmap; 929 930 for (n = 0, i = 0; n < nseg; n++) { 931 uint8_t first_sect, last_sect; 932 933 if ((n % SEGS_PER_INDIRECT_FRAME) == 0) { 934 /* Map indirect segments */ 935 if (segments) 936 kunmap_atomic(segments); 937 segments = kmap_atomic(pages[n/SEGS_PER_INDIRECT_FRAME]->page); 938 } 939 i = n % SEGS_PER_INDIRECT_FRAME; 940 941 pending_req->segments[n]->gref = segments[i].gref; 942 943 first_sect = READ_ONCE(segments[i].first_sect); 944 last_sect = READ_ONCE(segments[i].last_sect); 945 if (last_sect >= (XEN_PAGE_SIZE >> 9) || last_sect < first_sect) { 946 rc = -EINVAL; 947 goto unmap; 948 } 949 950 seg[n].nsec = last_sect - first_sect + 1; 951 seg[n].offset = first_sect << 9; 952 preq->nr_sects += seg[n].nsec; 953 } 954 955 unmap: 956 if (segments) 957 kunmap_atomic(segments); 958 xen_blkbk_unmap(ring, pages, indirect_grefs); 959 return rc; 960 } 961 962 static int dispatch_discard_io(struct xen_blkif_ring *ring, 963 struct blkif_request *req) 964 { 965 int err = 0; 966 int status = BLKIF_RSP_OKAY; 967 struct xen_blkif *blkif = ring->blkif; 968 struct block_device *bdev = blkif->vbd.bdev; 969 unsigned long secure; 970 struct phys_req preq; 971 972 xen_blkif_get(blkif); 973 974 preq.sector_number = req->u.discard.sector_number; 975 preq.nr_sects = req->u.discard.nr_sectors; 976 977 err = xen_vbd_translate(&preq, blkif, REQ_OP_WRITE); 978 if (err) { 979 pr_warn("access denied: DISCARD [%llu->%llu] on dev=%04x\n", 980 preq.sector_number, 981 preq.sector_number + preq.nr_sects, blkif->vbd.pdevice); 982 goto fail_response; 983 } 984 ring->st_ds_req++; 985 986 secure = (blkif->vbd.discard_secure && 987 (req->u.discard.flag & BLKIF_DISCARD_SECURE)) ? 988 BLKDEV_DISCARD_SECURE : 0; 989 990 err = blkdev_issue_discard(bdev, req->u.discard.sector_number, 991 req->u.discard.nr_sectors, 992 GFP_KERNEL, secure); 993 fail_response: 994 if (err == -EOPNOTSUPP) { 995 pr_debug("discard op failed, not supported\n"); 996 status = BLKIF_RSP_EOPNOTSUPP; 997 } else if (err) 998 status = BLKIF_RSP_ERROR; 999 1000 make_response(ring, req->u.discard.id, req->operation, status); 1001 xen_blkif_put(blkif); 1002 return err; 1003 } 1004 1005 static int dispatch_other_io(struct xen_blkif_ring *ring, 1006 struct blkif_request *req, 1007 struct pending_req *pending_req) 1008 { 1009 free_req(ring, pending_req); 1010 make_response(ring, req->u.other.id, req->operation, 1011 BLKIF_RSP_EOPNOTSUPP); 1012 return -EIO; 1013 } 1014 1015 static void xen_blk_drain_io(struct xen_blkif_ring *ring) 1016 { 1017 struct xen_blkif *blkif = ring->blkif; 1018 1019 atomic_set(&blkif->drain, 1); 1020 do { 1021 if (atomic_read(&ring->inflight) == 0) 1022 break; 1023 wait_for_completion_interruptible_timeout( 1024 &blkif->drain_complete, HZ); 1025 1026 if (!atomic_read(&blkif->drain)) 1027 break; 1028 } while (!kthread_should_stop()); 1029 atomic_set(&blkif->drain, 0); 1030 } 1031 1032 static void __end_block_io_op(struct pending_req *pending_req, 1033 blk_status_t error) 1034 { 1035 /* An error fails the entire request. */ 1036 if (pending_req->operation == BLKIF_OP_FLUSH_DISKCACHE && 1037 error == BLK_STS_NOTSUPP) { 1038 pr_debug("flush diskcache op failed, not supported\n"); 1039 xen_blkbk_flush_diskcache(XBT_NIL, pending_req->ring->blkif->be, 0); 1040 pending_req->status = BLKIF_RSP_EOPNOTSUPP; 1041 } else if (pending_req->operation == BLKIF_OP_WRITE_BARRIER && 1042 error == BLK_STS_NOTSUPP) { 1043 pr_debug("write barrier op failed, not supported\n"); 1044 xen_blkbk_barrier(XBT_NIL, pending_req->ring->blkif->be, 0); 1045 pending_req->status = BLKIF_RSP_EOPNOTSUPP; 1046 } else if (error) { 1047 pr_debug("Buffer not up-to-date at end of operation," 1048 " error=%d\n", error); 1049 pending_req->status = BLKIF_RSP_ERROR; 1050 } 1051 1052 /* 1053 * If all of the bio's have completed it is time to unmap 1054 * the grant references associated with 'request' and provide 1055 * the proper response on the ring. 1056 */ 1057 if (atomic_dec_and_test(&pending_req->pendcnt)) 1058 xen_blkbk_unmap_and_respond(pending_req); 1059 } 1060 1061 /* 1062 * bio callback. 1063 */ 1064 static void end_block_io_op(struct bio *bio) 1065 { 1066 __end_block_io_op(bio->bi_private, bio->bi_status); 1067 bio_put(bio); 1068 } 1069 1070 1071 1072 /* 1073 * Function to copy the from the ring buffer the 'struct blkif_request' 1074 * (which has the sectors we want, number of them, grant references, etc), 1075 * and transmute it to the block API to hand it over to the proper block disk. 1076 */ 1077 static int 1078 __do_block_io_op(struct xen_blkif_ring *ring, unsigned int *eoi_flags) 1079 { 1080 union blkif_back_rings *blk_rings = &ring->blk_rings; 1081 struct blkif_request req; 1082 struct pending_req *pending_req; 1083 RING_IDX rc, rp; 1084 int more_to_do = 0; 1085 1086 rc = blk_rings->common.req_cons; 1087 rp = blk_rings->common.sring->req_prod; 1088 rmb(); /* Ensure we see queued requests up to 'rp'. */ 1089 1090 if (RING_REQUEST_PROD_OVERFLOW(&blk_rings->common, rp)) { 1091 rc = blk_rings->common.rsp_prod_pvt; 1092 pr_warn("Frontend provided bogus ring requests (%d - %d = %d). Halting ring processing on dev=%04x\n", 1093 rp, rc, rp - rc, ring->blkif->vbd.pdevice); 1094 return -EACCES; 1095 } 1096 while (rc != rp) { 1097 1098 if (RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, rc)) 1099 break; 1100 1101 /* We've seen a request, so clear spurious eoi flag. */ 1102 *eoi_flags &= ~XEN_EOI_FLAG_SPURIOUS; 1103 1104 if (kthread_should_stop()) { 1105 more_to_do = 1; 1106 break; 1107 } 1108 1109 pending_req = alloc_req(ring); 1110 if (NULL == pending_req) { 1111 ring->st_oo_req++; 1112 more_to_do = 1; 1113 break; 1114 } 1115 1116 switch (ring->blkif->blk_protocol) { 1117 case BLKIF_PROTOCOL_NATIVE: 1118 memcpy(&req, RING_GET_REQUEST(&blk_rings->native, rc), sizeof(req)); 1119 break; 1120 case BLKIF_PROTOCOL_X86_32: 1121 blkif_get_x86_32_req(&req, RING_GET_REQUEST(&blk_rings->x86_32, rc)); 1122 break; 1123 case BLKIF_PROTOCOL_X86_64: 1124 blkif_get_x86_64_req(&req, RING_GET_REQUEST(&blk_rings->x86_64, rc)); 1125 break; 1126 default: 1127 BUG(); 1128 } 1129 blk_rings->common.req_cons = ++rc; /* before make_response() */ 1130 1131 /* Apply all sanity checks to /private copy/ of request. */ 1132 barrier(); 1133 1134 switch (req.operation) { 1135 case BLKIF_OP_READ: 1136 case BLKIF_OP_WRITE: 1137 case BLKIF_OP_WRITE_BARRIER: 1138 case BLKIF_OP_FLUSH_DISKCACHE: 1139 case BLKIF_OP_INDIRECT: 1140 if (dispatch_rw_block_io(ring, &req, pending_req)) 1141 goto done; 1142 break; 1143 case BLKIF_OP_DISCARD: 1144 free_req(ring, pending_req); 1145 if (dispatch_discard_io(ring, &req)) 1146 goto done; 1147 break; 1148 default: 1149 if (dispatch_other_io(ring, &req, pending_req)) 1150 goto done; 1151 break; 1152 } 1153 1154 /* Yield point for this unbounded loop. */ 1155 cond_resched(); 1156 } 1157 done: 1158 return more_to_do; 1159 } 1160 1161 static int 1162 do_block_io_op(struct xen_blkif_ring *ring, unsigned int *eoi_flags) 1163 { 1164 union blkif_back_rings *blk_rings = &ring->blk_rings; 1165 int more_to_do; 1166 1167 do { 1168 more_to_do = __do_block_io_op(ring, eoi_flags); 1169 if (more_to_do) 1170 break; 1171 1172 RING_FINAL_CHECK_FOR_REQUESTS(&blk_rings->common, more_to_do); 1173 } while (more_to_do); 1174 1175 return more_to_do; 1176 } 1177 /* 1178 * Transmutation of the 'struct blkif_request' to a proper 'struct bio' 1179 * and call the 'submit_bio' to pass it to the underlying storage. 1180 */ 1181 static int dispatch_rw_block_io(struct xen_blkif_ring *ring, 1182 struct blkif_request *req, 1183 struct pending_req *pending_req) 1184 { 1185 struct phys_req preq; 1186 struct seg_buf *seg = pending_req->seg; 1187 unsigned int nseg; 1188 struct bio *bio = NULL; 1189 struct bio **biolist = pending_req->biolist; 1190 int i, nbio = 0; 1191 int operation; 1192 int operation_flags = 0; 1193 struct blk_plug plug; 1194 bool drain = false; 1195 struct grant_page **pages = pending_req->segments; 1196 unsigned short req_operation; 1197 1198 req_operation = req->operation == BLKIF_OP_INDIRECT ? 1199 req->u.indirect.indirect_op : req->operation; 1200 1201 if ((req->operation == BLKIF_OP_INDIRECT) && 1202 (req_operation != BLKIF_OP_READ) && 1203 (req_operation != BLKIF_OP_WRITE)) { 1204 pr_debug("Invalid indirect operation (%u)\n", req_operation); 1205 goto fail_response; 1206 } 1207 1208 switch (req_operation) { 1209 case BLKIF_OP_READ: 1210 ring->st_rd_req++; 1211 operation = REQ_OP_READ; 1212 break; 1213 case BLKIF_OP_WRITE: 1214 ring->st_wr_req++; 1215 operation = REQ_OP_WRITE; 1216 operation_flags = REQ_SYNC | REQ_IDLE; 1217 break; 1218 case BLKIF_OP_WRITE_BARRIER: 1219 drain = true; 1220 fallthrough; 1221 case BLKIF_OP_FLUSH_DISKCACHE: 1222 ring->st_f_req++; 1223 operation = REQ_OP_WRITE; 1224 operation_flags = REQ_PREFLUSH; 1225 break; 1226 default: 1227 operation = 0; /* make gcc happy */ 1228 goto fail_response; 1229 break; 1230 } 1231 1232 /* Check that the number of segments is sane. */ 1233 nseg = req->operation == BLKIF_OP_INDIRECT ? 1234 req->u.indirect.nr_segments : req->u.rw.nr_segments; 1235 1236 if (unlikely(nseg == 0 && operation_flags != REQ_PREFLUSH) || 1237 unlikely((req->operation != BLKIF_OP_INDIRECT) && 1238 (nseg > BLKIF_MAX_SEGMENTS_PER_REQUEST)) || 1239 unlikely((req->operation == BLKIF_OP_INDIRECT) && 1240 (nseg > MAX_INDIRECT_SEGMENTS))) { 1241 pr_debug("Bad number of segments in request (%d)\n", nseg); 1242 /* Haven't submitted any bio's yet. */ 1243 goto fail_response; 1244 } 1245 1246 preq.nr_sects = 0; 1247 1248 pending_req->ring = ring; 1249 pending_req->id = req->u.rw.id; 1250 pending_req->operation = req_operation; 1251 pending_req->status = BLKIF_RSP_OKAY; 1252 pending_req->nr_segs = nseg; 1253 1254 if (req->operation != BLKIF_OP_INDIRECT) { 1255 preq.dev = req->u.rw.handle; 1256 preq.sector_number = req->u.rw.sector_number; 1257 for (i = 0; i < nseg; i++) { 1258 pages[i]->gref = req->u.rw.seg[i].gref; 1259 seg[i].nsec = req->u.rw.seg[i].last_sect - 1260 req->u.rw.seg[i].first_sect + 1; 1261 seg[i].offset = (req->u.rw.seg[i].first_sect << 9); 1262 if ((req->u.rw.seg[i].last_sect >= (XEN_PAGE_SIZE >> 9)) || 1263 (req->u.rw.seg[i].last_sect < 1264 req->u.rw.seg[i].first_sect)) 1265 goto fail_response; 1266 preq.nr_sects += seg[i].nsec; 1267 } 1268 } else { 1269 preq.dev = req->u.indirect.handle; 1270 preq.sector_number = req->u.indirect.sector_number; 1271 if (xen_blkbk_parse_indirect(req, pending_req, seg, &preq)) 1272 goto fail_response; 1273 } 1274 1275 if (xen_vbd_translate(&preq, ring->blkif, operation) != 0) { 1276 pr_debug("access denied: %s of [%llu,%llu] on dev=%04x\n", 1277 operation == REQ_OP_READ ? "read" : "write", 1278 preq.sector_number, 1279 preq.sector_number + preq.nr_sects, 1280 ring->blkif->vbd.pdevice); 1281 goto fail_response; 1282 } 1283 1284 /* 1285 * This check _MUST_ be done after xen_vbd_translate as the preq.bdev 1286 * is set there. 1287 */ 1288 for (i = 0; i < nseg; i++) { 1289 if (((int)preq.sector_number|(int)seg[i].nsec) & 1290 ((bdev_logical_block_size(preq.bdev) >> 9) - 1)) { 1291 pr_debug("Misaligned I/O request from domain %d\n", 1292 ring->blkif->domid); 1293 goto fail_response; 1294 } 1295 } 1296 1297 /* Wait on all outstanding I/O's and once that has been completed 1298 * issue the flush. 1299 */ 1300 if (drain) 1301 xen_blk_drain_io(pending_req->ring); 1302 1303 /* 1304 * If we have failed at this point, we need to undo the M2P override, 1305 * set gnttab_set_unmap_op on all of the grant references and perform 1306 * the hypercall to unmap the grants - that is all done in 1307 * xen_blkbk_unmap. 1308 */ 1309 if (xen_blkbk_map_seg(pending_req)) 1310 goto fail_flush; 1311 1312 /* 1313 * This corresponding xen_blkif_put is done in __end_block_io_op, or 1314 * below (in "!bio") if we are handling a BLKIF_OP_DISCARD. 1315 */ 1316 xen_blkif_get(ring->blkif); 1317 atomic_inc(&ring->inflight); 1318 1319 for (i = 0; i < nseg; i++) { 1320 while ((bio == NULL) || 1321 (bio_add_page(bio, 1322 pages[i]->page, 1323 seg[i].nsec << 9, 1324 seg[i].offset) == 0)) { 1325 1326 int nr_iovecs = min_t(int, (nseg-i), BIO_MAX_PAGES); 1327 bio = bio_alloc(GFP_KERNEL, nr_iovecs); 1328 if (unlikely(bio == NULL)) 1329 goto fail_put_bio; 1330 1331 biolist[nbio++] = bio; 1332 bio_set_dev(bio, preq.bdev); 1333 bio->bi_private = pending_req; 1334 bio->bi_end_io = end_block_io_op; 1335 bio->bi_iter.bi_sector = preq.sector_number; 1336 bio_set_op_attrs(bio, operation, operation_flags); 1337 } 1338 1339 preq.sector_number += seg[i].nsec; 1340 } 1341 1342 /* This will be hit if the operation was a flush or discard. */ 1343 if (!bio) { 1344 BUG_ON(operation_flags != REQ_PREFLUSH); 1345 1346 bio = bio_alloc(GFP_KERNEL, 0); 1347 if (unlikely(bio == NULL)) 1348 goto fail_put_bio; 1349 1350 biolist[nbio++] = bio; 1351 bio_set_dev(bio, preq.bdev); 1352 bio->bi_private = pending_req; 1353 bio->bi_end_io = end_block_io_op; 1354 bio_set_op_attrs(bio, operation, operation_flags); 1355 } 1356 1357 atomic_set(&pending_req->pendcnt, nbio); 1358 blk_start_plug(&plug); 1359 1360 for (i = 0; i < nbio; i++) 1361 submit_bio(biolist[i]); 1362 1363 /* Let the I/Os go.. */ 1364 blk_finish_plug(&plug); 1365 1366 if (operation == REQ_OP_READ) 1367 ring->st_rd_sect += preq.nr_sects; 1368 else if (operation == REQ_OP_WRITE) 1369 ring->st_wr_sect += preq.nr_sects; 1370 1371 return 0; 1372 1373 fail_flush: 1374 xen_blkbk_unmap(ring, pending_req->segments, 1375 pending_req->nr_segs); 1376 fail_response: 1377 /* Haven't submitted any bio's yet. */ 1378 make_response(ring, req->u.rw.id, req_operation, BLKIF_RSP_ERROR); 1379 free_req(ring, pending_req); 1380 msleep(1); /* back off a bit */ 1381 return -EIO; 1382 1383 fail_put_bio: 1384 for (i = 0; i < nbio; i++) 1385 bio_put(biolist[i]); 1386 atomic_set(&pending_req->pendcnt, 1); 1387 __end_block_io_op(pending_req, BLK_STS_RESOURCE); 1388 msleep(1); /* back off a bit */ 1389 return -EIO; 1390 } 1391 1392 1393 1394 /* 1395 * Put a response on the ring on how the operation fared. 1396 */ 1397 static void make_response(struct xen_blkif_ring *ring, u64 id, 1398 unsigned short op, int st) 1399 { 1400 struct blkif_response *resp; 1401 unsigned long flags; 1402 union blkif_back_rings *blk_rings; 1403 int notify; 1404 1405 spin_lock_irqsave(&ring->blk_ring_lock, flags); 1406 blk_rings = &ring->blk_rings; 1407 /* Place on the response ring for the relevant domain. */ 1408 switch (ring->blkif->blk_protocol) { 1409 case BLKIF_PROTOCOL_NATIVE: 1410 resp = RING_GET_RESPONSE(&blk_rings->native, 1411 blk_rings->native.rsp_prod_pvt); 1412 break; 1413 case BLKIF_PROTOCOL_X86_32: 1414 resp = RING_GET_RESPONSE(&blk_rings->x86_32, 1415 blk_rings->x86_32.rsp_prod_pvt); 1416 break; 1417 case BLKIF_PROTOCOL_X86_64: 1418 resp = RING_GET_RESPONSE(&blk_rings->x86_64, 1419 blk_rings->x86_64.rsp_prod_pvt); 1420 break; 1421 default: 1422 BUG(); 1423 } 1424 1425 resp->id = id; 1426 resp->operation = op; 1427 resp->status = st; 1428 1429 blk_rings->common.rsp_prod_pvt++; 1430 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&blk_rings->common, notify); 1431 spin_unlock_irqrestore(&ring->blk_ring_lock, flags); 1432 if (notify) 1433 notify_remote_via_irq(ring->irq); 1434 } 1435 1436 static int __init xen_blkif_init(void) 1437 { 1438 int rc = 0; 1439 1440 if (!xen_domain()) 1441 return -ENODEV; 1442 1443 if (xen_blkif_max_ring_order > XENBUS_MAX_RING_GRANT_ORDER) { 1444 pr_info("Invalid max_ring_order (%d), will use default max: %d.\n", 1445 xen_blkif_max_ring_order, XENBUS_MAX_RING_GRANT_ORDER); 1446 xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER; 1447 } 1448 1449 if (xenblk_max_queues == 0) 1450 xenblk_max_queues = num_online_cpus(); 1451 1452 rc = xen_blkif_interface_init(); 1453 if (rc) 1454 goto failed_init; 1455 1456 rc = xen_blkif_xenbus_init(); 1457 if (rc) 1458 goto failed_init; 1459 1460 failed_init: 1461 return rc; 1462 } 1463 1464 module_init(xen_blkif_init); 1465 1466 static void __exit xen_blkif_fini(void) 1467 { 1468 xen_blkif_xenbus_fini(); 1469 xen_blkif_interface_fini(); 1470 } 1471 1472 module_exit(xen_blkif_fini); 1473 1474 MODULE_LICENSE("Dual BSD/GPL"); 1475 MODULE_ALIAS("xen-backend:vbd"); 1476