1 // SPDX-License-Identifier: GPL-2.0-only 2 //#define DEBUG 3 #include <linux/spinlock.h> 4 #include <linux/slab.h> 5 #include <linux/blkdev.h> 6 #include <linux/hdreg.h> 7 #include <linux/module.h> 8 #include <linux/mutex.h> 9 #include <linux/interrupt.h> 10 #include <linux/virtio.h> 11 #include <linux/virtio_blk.h> 12 #include <linux/scatterlist.h> 13 #include <linux/string_helpers.h> 14 #include <linux/idr.h> 15 #include <linux/blk-mq.h> 16 #include <linux/blk-mq-virtio.h> 17 #include <linux/numa.h> 18 #include <uapi/linux/virtio_ring.h> 19 20 #define PART_BITS 4 21 #define VQ_NAME_LEN 16 22 #define MAX_DISCARD_SEGMENTS 256u 23 24 static int major; 25 static DEFINE_IDA(vd_index_ida); 26 27 static struct workqueue_struct *virtblk_wq; 28 29 struct virtio_blk_vq { 30 struct virtqueue *vq; 31 spinlock_t lock; 32 char name[VQ_NAME_LEN]; 33 } ____cacheline_aligned_in_smp; 34 35 struct virtio_blk { 36 struct virtio_device *vdev; 37 38 /* The disk structure for the kernel. */ 39 struct gendisk *disk; 40 41 /* Block layer tags. */ 42 struct blk_mq_tag_set tag_set; 43 44 /* Process context for config space updates */ 45 struct work_struct config_work; 46 47 /* What host tells us, plus 2 for header & tailer. */ 48 unsigned int sg_elems; 49 50 /* Ida index - used to track minor number allocations. */ 51 int index; 52 53 /* num of vqs */ 54 int num_vqs; 55 struct virtio_blk_vq *vqs; 56 }; 57 58 struct virtblk_req { 59 struct virtio_blk_outhdr out_hdr; 60 u8 status; 61 struct scatterlist sg[]; 62 }; 63 64 static inline blk_status_t virtblk_result(struct virtblk_req *vbr) 65 { 66 switch (vbr->status) { 67 case VIRTIO_BLK_S_OK: 68 return BLK_STS_OK; 69 case VIRTIO_BLK_S_UNSUPP: 70 return BLK_STS_NOTSUPP; 71 default: 72 return BLK_STS_IOERR; 73 } 74 } 75 76 static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr, 77 struct scatterlist *data_sg, bool have_data) 78 { 79 struct scatterlist hdr, status, *sgs[3]; 80 unsigned int num_out = 0, num_in = 0; 81 82 sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr)); 83 sgs[num_out++] = &hdr; 84 85 if (have_data) { 86 if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT)) 87 sgs[num_out++] = data_sg; 88 else 89 sgs[num_out + num_in++] = data_sg; 90 } 91 92 sg_init_one(&status, &vbr->status, sizeof(vbr->status)); 93 sgs[num_out + num_in++] = &status; 94 95 return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC); 96 } 97 98 static int virtblk_setup_discard_write_zeroes(struct request *req, bool unmap) 99 { 100 unsigned short segments = blk_rq_nr_discard_segments(req); 101 unsigned short n = 0; 102 struct virtio_blk_discard_write_zeroes *range; 103 struct bio *bio; 104 u32 flags = 0; 105 106 if (unmap) 107 flags |= VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP; 108 109 range = kmalloc_array(segments, sizeof(*range), GFP_ATOMIC); 110 if (!range) 111 return -ENOMEM; 112 113 __rq_for_each_bio(bio, req) { 114 u64 sector = bio->bi_iter.bi_sector; 115 u32 num_sectors = bio->bi_iter.bi_size >> SECTOR_SHIFT; 116 117 range[n].flags = cpu_to_le32(flags); 118 range[n].num_sectors = cpu_to_le32(num_sectors); 119 range[n].sector = cpu_to_le64(sector); 120 n++; 121 } 122 123 req->special_vec.bv_page = virt_to_page(range); 124 req->special_vec.bv_offset = offset_in_page(range); 125 req->special_vec.bv_len = sizeof(*range) * segments; 126 req->rq_flags |= RQF_SPECIAL_PAYLOAD; 127 128 return 0; 129 } 130 131 static inline void virtblk_request_done(struct request *req) 132 { 133 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); 134 135 if (req->rq_flags & RQF_SPECIAL_PAYLOAD) { 136 kfree(page_address(req->special_vec.bv_page) + 137 req->special_vec.bv_offset); 138 } 139 140 blk_mq_end_request(req, virtblk_result(vbr)); 141 } 142 143 static void virtblk_done(struct virtqueue *vq) 144 { 145 struct virtio_blk *vblk = vq->vdev->priv; 146 bool req_done = false; 147 int qid = vq->index; 148 struct virtblk_req *vbr; 149 unsigned long flags; 150 unsigned int len; 151 152 spin_lock_irqsave(&vblk->vqs[qid].lock, flags); 153 do { 154 virtqueue_disable_cb(vq); 155 while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) { 156 struct request *req = blk_mq_rq_from_pdu(vbr); 157 158 blk_mq_complete_request(req); 159 req_done = true; 160 } 161 if (unlikely(virtqueue_is_broken(vq))) 162 break; 163 } while (!virtqueue_enable_cb(vq)); 164 165 /* In case queue is stopped waiting for more buffers. */ 166 if (req_done) 167 blk_mq_start_stopped_hw_queues(vblk->disk->queue, true); 168 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); 169 } 170 171 static void virtio_commit_rqs(struct blk_mq_hw_ctx *hctx) 172 { 173 struct virtio_blk *vblk = hctx->queue->queuedata; 174 struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num]; 175 bool kick; 176 177 spin_lock_irq(&vq->lock); 178 kick = virtqueue_kick_prepare(vq->vq); 179 spin_unlock_irq(&vq->lock); 180 181 if (kick) 182 virtqueue_notify(vq->vq); 183 } 184 185 static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx, 186 const struct blk_mq_queue_data *bd) 187 { 188 struct virtio_blk *vblk = hctx->queue->queuedata; 189 struct request *req = bd->rq; 190 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); 191 unsigned long flags; 192 unsigned int num; 193 int qid = hctx->queue_num; 194 int err; 195 bool notify = false; 196 bool unmap = false; 197 u32 type; 198 199 BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems); 200 201 switch (req_op(req)) { 202 case REQ_OP_READ: 203 case REQ_OP_WRITE: 204 type = 0; 205 break; 206 case REQ_OP_FLUSH: 207 type = VIRTIO_BLK_T_FLUSH; 208 break; 209 case REQ_OP_DISCARD: 210 type = VIRTIO_BLK_T_DISCARD; 211 break; 212 case REQ_OP_WRITE_ZEROES: 213 type = VIRTIO_BLK_T_WRITE_ZEROES; 214 unmap = !(req->cmd_flags & REQ_NOUNMAP); 215 break; 216 case REQ_OP_DRV_IN: 217 type = VIRTIO_BLK_T_GET_ID; 218 break; 219 default: 220 WARN_ON_ONCE(1); 221 return BLK_STS_IOERR; 222 } 223 224 vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, type); 225 vbr->out_hdr.sector = type ? 226 0 : cpu_to_virtio64(vblk->vdev, blk_rq_pos(req)); 227 vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(req)); 228 229 blk_mq_start_request(req); 230 231 if (type == VIRTIO_BLK_T_DISCARD || type == VIRTIO_BLK_T_WRITE_ZEROES) { 232 err = virtblk_setup_discard_write_zeroes(req, unmap); 233 if (err) 234 return BLK_STS_RESOURCE; 235 } 236 237 num = blk_rq_map_sg(hctx->queue, req, vbr->sg); 238 if (num) { 239 if (rq_data_dir(req) == WRITE) 240 vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_OUT); 241 else 242 vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_IN); 243 } 244 245 spin_lock_irqsave(&vblk->vqs[qid].lock, flags); 246 err = virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg, num); 247 if (err) { 248 virtqueue_kick(vblk->vqs[qid].vq); 249 /* Don't stop the queue if -ENOMEM: we may have failed to 250 * bounce the buffer due to global resource outage. 251 */ 252 if (err == -ENOSPC) 253 blk_mq_stop_hw_queue(hctx); 254 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); 255 switch (err) { 256 case -ENOSPC: 257 return BLK_STS_DEV_RESOURCE; 258 case -ENOMEM: 259 return BLK_STS_RESOURCE; 260 default: 261 return BLK_STS_IOERR; 262 } 263 } 264 265 if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq)) 266 notify = true; 267 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); 268 269 if (notify) 270 virtqueue_notify(vblk->vqs[qid].vq); 271 return BLK_STS_OK; 272 } 273 274 /* return id (s/n) string for *disk to *id_str 275 */ 276 static int virtblk_get_id(struct gendisk *disk, char *id_str) 277 { 278 struct virtio_blk *vblk = disk->private_data; 279 struct request_queue *q = vblk->disk->queue; 280 struct request *req; 281 int err; 282 283 req = blk_get_request(q, REQ_OP_DRV_IN, 0); 284 if (IS_ERR(req)) 285 return PTR_ERR(req); 286 287 err = blk_rq_map_kern(q, req, id_str, VIRTIO_BLK_ID_BYTES, GFP_KERNEL); 288 if (err) 289 goto out; 290 291 blk_execute_rq(vblk->disk->queue, vblk->disk, req, false); 292 err = blk_status_to_errno(virtblk_result(blk_mq_rq_to_pdu(req))); 293 out: 294 blk_put_request(req); 295 return err; 296 } 297 298 /* We provide getgeo only to please some old bootloader/partitioning tools */ 299 static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo) 300 { 301 struct virtio_blk *vblk = bd->bd_disk->private_data; 302 303 /* see if the host passed in geometry config */ 304 if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) { 305 virtio_cread(vblk->vdev, struct virtio_blk_config, 306 geometry.cylinders, &geo->cylinders); 307 virtio_cread(vblk->vdev, struct virtio_blk_config, 308 geometry.heads, &geo->heads); 309 virtio_cread(vblk->vdev, struct virtio_blk_config, 310 geometry.sectors, &geo->sectors); 311 } else { 312 /* some standard values, similar to sd */ 313 geo->heads = 1 << 6; 314 geo->sectors = 1 << 5; 315 geo->cylinders = get_capacity(bd->bd_disk) >> 11; 316 } 317 return 0; 318 } 319 320 static const struct block_device_operations virtblk_fops = { 321 .owner = THIS_MODULE, 322 .getgeo = virtblk_getgeo, 323 }; 324 325 static int index_to_minor(int index) 326 { 327 return index << PART_BITS; 328 } 329 330 static int minor_to_index(int minor) 331 { 332 return minor >> PART_BITS; 333 } 334 335 static ssize_t serial_show(struct device *dev, 336 struct device_attribute *attr, char *buf) 337 { 338 struct gendisk *disk = dev_to_disk(dev); 339 int err; 340 341 /* sysfs gives us a PAGE_SIZE buffer */ 342 BUILD_BUG_ON(PAGE_SIZE < VIRTIO_BLK_ID_BYTES); 343 344 buf[VIRTIO_BLK_ID_BYTES] = '\0'; 345 err = virtblk_get_id(disk, buf); 346 if (!err) 347 return strlen(buf); 348 349 if (err == -EIO) /* Unsupported? Make it empty. */ 350 return 0; 351 352 return err; 353 } 354 355 static DEVICE_ATTR_RO(serial); 356 357 /* The queue's logical block size must be set before calling this */ 358 static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize) 359 { 360 struct virtio_device *vdev = vblk->vdev; 361 struct request_queue *q = vblk->disk->queue; 362 char cap_str_2[10], cap_str_10[10]; 363 unsigned long long nblocks; 364 u64 capacity; 365 366 /* Host must always specify the capacity. */ 367 virtio_cread(vdev, struct virtio_blk_config, capacity, &capacity); 368 369 /* If capacity is too big, truncate with warning. */ 370 if ((sector_t)capacity != capacity) { 371 dev_warn(&vdev->dev, "Capacity %llu too large: truncating\n", 372 (unsigned long long)capacity); 373 capacity = (sector_t)-1; 374 } 375 376 nblocks = DIV_ROUND_UP_ULL(capacity, queue_logical_block_size(q) >> 9); 377 378 string_get_size(nblocks, queue_logical_block_size(q), 379 STRING_UNITS_2, cap_str_2, sizeof(cap_str_2)); 380 string_get_size(nblocks, queue_logical_block_size(q), 381 STRING_UNITS_10, cap_str_10, sizeof(cap_str_10)); 382 383 dev_notice(&vdev->dev, 384 "[%s] %s%llu %d-byte logical blocks (%s/%s)\n", 385 vblk->disk->disk_name, 386 resize ? "new size: " : "", 387 nblocks, 388 queue_logical_block_size(q), 389 cap_str_10, 390 cap_str_2); 391 392 set_capacity_revalidate_and_notify(vblk->disk, capacity, true); 393 } 394 395 static void virtblk_config_changed_work(struct work_struct *work) 396 { 397 struct virtio_blk *vblk = 398 container_of(work, struct virtio_blk, config_work); 399 400 virtblk_update_capacity(vblk, true); 401 } 402 403 static void virtblk_config_changed(struct virtio_device *vdev) 404 { 405 struct virtio_blk *vblk = vdev->priv; 406 407 queue_work(virtblk_wq, &vblk->config_work); 408 } 409 410 static int init_vq(struct virtio_blk *vblk) 411 { 412 int err; 413 int i; 414 vq_callback_t **callbacks; 415 const char **names; 416 struct virtqueue **vqs; 417 unsigned short num_vqs; 418 struct virtio_device *vdev = vblk->vdev; 419 struct irq_affinity desc = { 0, }; 420 421 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_MQ, 422 struct virtio_blk_config, num_queues, 423 &num_vqs); 424 if (err) 425 num_vqs = 1; 426 427 num_vqs = min_t(unsigned int, nr_cpu_ids, num_vqs); 428 429 vblk->vqs = kmalloc_array(num_vqs, sizeof(*vblk->vqs), GFP_KERNEL); 430 if (!vblk->vqs) 431 return -ENOMEM; 432 433 names = kmalloc_array(num_vqs, sizeof(*names), GFP_KERNEL); 434 callbacks = kmalloc_array(num_vqs, sizeof(*callbacks), GFP_KERNEL); 435 vqs = kmalloc_array(num_vqs, sizeof(*vqs), GFP_KERNEL); 436 if (!names || !callbacks || !vqs) { 437 err = -ENOMEM; 438 goto out; 439 } 440 441 for (i = 0; i < num_vqs; i++) { 442 callbacks[i] = virtblk_done; 443 snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%d", i); 444 names[i] = vblk->vqs[i].name; 445 } 446 447 /* Discover virtqueues and write information to configuration. */ 448 err = virtio_find_vqs(vdev, num_vqs, vqs, callbacks, names, &desc); 449 if (err) 450 goto out; 451 452 for (i = 0; i < num_vqs; i++) { 453 spin_lock_init(&vblk->vqs[i].lock); 454 vblk->vqs[i].vq = vqs[i]; 455 } 456 vblk->num_vqs = num_vqs; 457 458 out: 459 kfree(vqs); 460 kfree(callbacks); 461 kfree(names); 462 if (err) 463 kfree(vblk->vqs); 464 return err; 465 } 466 467 /* 468 * Legacy naming scheme used for virtio devices. We are stuck with it for 469 * virtio blk but don't ever use it for any new driver. 470 */ 471 static int virtblk_name_format(char *prefix, int index, char *buf, int buflen) 472 { 473 const int base = 'z' - 'a' + 1; 474 char *begin = buf + strlen(prefix); 475 char *end = buf + buflen; 476 char *p; 477 int unit; 478 479 p = end - 1; 480 *p = '\0'; 481 unit = base; 482 do { 483 if (p == begin) 484 return -EINVAL; 485 *--p = 'a' + (index % unit); 486 index = (index / unit) - 1; 487 } while (index >= 0); 488 489 memmove(begin, p, end - p); 490 memcpy(buf, prefix, strlen(prefix)); 491 492 return 0; 493 } 494 495 static int virtblk_get_cache_mode(struct virtio_device *vdev) 496 { 497 u8 writeback; 498 int err; 499 500 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE, 501 struct virtio_blk_config, wce, 502 &writeback); 503 504 /* 505 * If WCE is not configurable and flush is not available, 506 * assume no writeback cache is in use. 507 */ 508 if (err) 509 writeback = virtio_has_feature(vdev, VIRTIO_BLK_F_FLUSH); 510 511 return writeback; 512 } 513 514 static void virtblk_update_cache_mode(struct virtio_device *vdev) 515 { 516 u8 writeback = virtblk_get_cache_mode(vdev); 517 struct virtio_blk *vblk = vdev->priv; 518 519 blk_queue_write_cache(vblk->disk->queue, writeback, false); 520 revalidate_disk(vblk->disk); 521 } 522 523 static const char *const virtblk_cache_types[] = { 524 "write through", "write back" 525 }; 526 527 static ssize_t 528 cache_type_store(struct device *dev, struct device_attribute *attr, 529 const char *buf, size_t count) 530 { 531 struct gendisk *disk = dev_to_disk(dev); 532 struct virtio_blk *vblk = disk->private_data; 533 struct virtio_device *vdev = vblk->vdev; 534 int i; 535 536 BUG_ON(!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_CONFIG_WCE)); 537 i = sysfs_match_string(virtblk_cache_types, buf); 538 if (i < 0) 539 return i; 540 541 virtio_cwrite8(vdev, offsetof(struct virtio_blk_config, wce), i); 542 virtblk_update_cache_mode(vdev); 543 return count; 544 } 545 546 static ssize_t 547 cache_type_show(struct device *dev, struct device_attribute *attr, char *buf) 548 { 549 struct gendisk *disk = dev_to_disk(dev); 550 struct virtio_blk *vblk = disk->private_data; 551 u8 writeback = virtblk_get_cache_mode(vblk->vdev); 552 553 BUG_ON(writeback >= ARRAY_SIZE(virtblk_cache_types)); 554 return snprintf(buf, 40, "%s\n", virtblk_cache_types[writeback]); 555 } 556 557 static DEVICE_ATTR_RW(cache_type); 558 559 static struct attribute *virtblk_attrs[] = { 560 &dev_attr_serial.attr, 561 &dev_attr_cache_type.attr, 562 NULL, 563 }; 564 565 static umode_t virtblk_attrs_are_visible(struct kobject *kobj, 566 struct attribute *a, int n) 567 { 568 struct device *dev = container_of(kobj, struct device, kobj); 569 struct gendisk *disk = dev_to_disk(dev); 570 struct virtio_blk *vblk = disk->private_data; 571 struct virtio_device *vdev = vblk->vdev; 572 573 if (a == &dev_attr_cache_type.attr && 574 !virtio_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE)) 575 return S_IRUGO; 576 577 return a->mode; 578 } 579 580 static const struct attribute_group virtblk_attr_group = { 581 .attrs = virtblk_attrs, 582 .is_visible = virtblk_attrs_are_visible, 583 }; 584 585 static const struct attribute_group *virtblk_attr_groups[] = { 586 &virtblk_attr_group, 587 NULL, 588 }; 589 590 static int virtblk_init_request(struct blk_mq_tag_set *set, struct request *rq, 591 unsigned int hctx_idx, unsigned int numa_node) 592 { 593 struct virtio_blk *vblk = set->driver_data; 594 struct virtblk_req *vbr = blk_mq_rq_to_pdu(rq); 595 596 sg_init_table(vbr->sg, vblk->sg_elems); 597 return 0; 598 } 599 600 static int virtblk_map_queues(struct blk_mq_tag_set *set) 601 { 602 struct virtio_blk *vblk = set->driver_data; 603 604 return blk_mq_virtio_map_queues(&set->map[HCTX_TYPE_DEFAULT], 605 vblk->vdev, 0); 606 } 607 608 static const struct blk_mq_ops virtio_mq_ops = { 609 .queue_rq = virtio_queue_rq, 610 .commit_rqs = virtio_commit_rqs, 611 .complete = virtblk_request_done, 612 .init_request = virtblk_init_request, 613 .map_queues = virtblk_map_queues, 614 }; 615 616 static unsigned int virtblk_queue_depth; 617 module_param_named(queue_depth, virtblk_queue_depth, uint, 0444); 618 619 static int virtblk_probe(struct virtio_device *vdev) 620 { 621 struct virtio_blk *vblk; 622 struct request_queue *q; 623 int err, index; 624 625 u32 v, blk_size, max_size, sg_elems, opt_io_size; 626 u16 min_io_size; 627 u8 physical_block_exp, alignment_offset; 628 629 if (!vdev->config->get) { 630 dev_err(&vdev->dev, "%s failure: config access disabled\n", 631 __func__); 632 return -EINVAL; 633 } 634 635 err = ida_simple_get(&vd_index_ida, 0, minor_to_index(1 << MINORBITS), 636 GFP_KERNEL); 637 if (err < 0) 638 goto out; 639 index = err; 640 641 /* We need to know how many segments before we allocate. */ 642 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SEG_MAX, 643 struct virtio_blk_config, seg_max, 644 &sg_elems); 645 646 /* We need at least one SG element, whatever they say. */ 647 if (err || !sg_elems) 648 sg_elems = 1; 649 650 /* We need an extra sg elements at head and tail. */ 651 sg_elems += 2; 652 vdev->priv = vblk = kmalloc(sizeof(*vblk), GFP_KERNEL); 653 if (!vblk) { 654 err = -ENOMEM; 655 goto out_free_index; 656 } 657 658 vblk->vdev = vdev; 659 vblk->sg_elems = sg_elems; 660 661 INIT_WORK(&vblk->config_work, virtblk_config_changed_work); 662 663 err = init_vq(vblk); 664 if (err) 665 goto out_free_vblk; 666 667 /* FIXME: How many partitions? How long is a piece of string? */ 668 vblk->disk = alloc_disk(1 << PART_BITS); 669 if (!vblk->disk) { 670 err = -ENOMEM; 671 goto out_free_vq; 672 } 673 674 /* Default queue sizing is to fill the ring. */ 675 if (!virtblk_queue_depth) { 676 virtblk_queue_depth = vblk->vqs[0].vq->num_free; 677 /* ... but without indirect descs, we use 2 descs per req */ 678 if (!virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) 679 virtblk_queue_depth /= 2; 680 } 681 682 memset(&vblk->tag_set, 0, sizeof(vblk->tag_set)); 683 vblk->tag_set.ops = &virtio_mq_ops; 684 vblk->tag_set.queue_depth = virtblk_queue_depth; 685 vblk->tag_set.numa_node = NUMA_NO_NODE; 686 vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; 687 vblk->tag_set.cmd_size = 688 sizeof(struct virtblk_req) + 689 sizeof(struct scatterlist) * sg_elems; 690 vblk->tag_set.driver_data = vblk; 691 vblk->tag_set.nr_hw_queues = vblk->num_vqs; 692 693 err = blk_mq_alloc_tag_set(&vblk->tag_set); 694 if (err) 695 goto out_put_disk; 696 697 q = blk_mq_init_queue(&vblk->tag_set); 698 if (IS_ERR(q)) { 699 err = -ENOMEM; 700 goto out_free_tags; 701 } 702 vblk->disk->queue = q; 703 704 q->queuedata = vblk; 705 706 virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN); 707 708 vblk->disk->major = major; 709 vblk->disk->first_minor = index_to_minor(index); 710 vblk->disk->private_data = vblk; 711 vblk->disk->fops = &virtblk_fops; 712 vblk->disk->flags |= GENHD_FL_EXT_DEVT; 713 vblk->index = index; 714 715 /* configure queue flush support */ 716 virtblk_update_cache_mode(vdev); 717 718 /* If disk is read-only in the host, the guest should obey */ 719 if (virtio_has_feature(vdev, VIRTIO_BLK_F_RO)) 720 set_disk_ro(vblk->disk, 1); 721 722 /* We can handle whatever the host told us to handle. */ 723 blk_queue_max_segments(q, vblk->sg_elems-2); 724 725 /* No real sector limit. */ 726 blk_queue_max_hw_sectors(q, -1U); 727 728 max_size = virtio_max_dma_size(vdev); 729 730 /* Host can optionally specify maximum segment size and number of 731 * segments. */ 732 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SIZE_MAX, 733 struct virtio_blk_config, size_max, &v); 734 if (!err) 735 max_size = min(max_size, v); 736 737 blk_queue_max_segment_size(q, max_size); 738 739 /* Host can optionally specify the block size of the device */ 740 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_BLK_SIZE, 741 struct virtio_blk_config, blk_size, 742 &blk_size); 743 if (!err) 744 blk_queue_logical_block_size(q, blk_size); 745 else 746 blk_size = queue_logical_block_size(q); 747 748 /* Use topology information if available */ 749 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 750 struct virtio_blk_config, physical_block_exp, 751 &physical_block_exp); 752 if (!err && physical_block_exp) 753 blk_queue_physical_block_size(q, 754 blk_size * (1 << physical_block_exp)); 755 756 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 757 struct virtio_blk_config, alignment_offset, 758 &alignment_offset); 759 if (!err && alignment_offset) 760 blk_queue_alignment_offset(q, blk_size * alignment_offset); 761 762 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 763 struct virtio_blk_config, min_io_size, 764 &min_io_size); 765 if (!err && min_io_size) 766 blk_queue_io_min(q, blk_size * min_io_size); 767 768 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 769 struct virtio_blk_config, opt_io_size, 770 &opt_io_size); 771 if (!err && opt_io_size) 772 blk_queue_io_opt(q, blk_size * opt_io_size); 773 774 if (virtio_has_feature(vdev, VIRTIO_BLK_F_DISCARD)) { 775 q->limits.discard_granularity = blk_size; 776 777 virtio_cread(vdev, struct virtio_blk_config, 778 discard_sector_alignment, &v); 779 q->limits.discard_alignment = v ? v << SECTOR_SHIFT : 0; 780 781 virtio_cread(vdev, struct virtio_blk_config, 782 max_discard_sectors, &v); 783 blk_queue_max_discard_sectors(q, v ? v : UINT_MAX); 784 785 virtio_cread(vdev, struct virtio_blk_config, max_discard_seg, 786 &v); 787 blk_queue_max_discard_segments(q, 788 min_not_zero(v, 789 MAX_DISCARD_SEGMENTS)); 790 791 blk_queue_flag_set(QUEUE_FLAG_DISCARD, q); 792 } 793 794 if (virtio_has_feature(vdev, VIRTIO_BLK_F_WRITE_ZEROES)) { 795 virtio_cread(vdev, struct virtio_blk_config, 796 max_write_zeroes_sectors, &v); 797 blk_queue_max_write_zeroes_sectors(q, v ? v : UINT_MAX); 798 } 799 800 virtblk_update_capacity(vblk, false); 801 virtio_device_ready(vdev); 802 803 device_add_disk(&vdev->dev, vblk->disk, virtblk_attr_groups); 804 return 0; 805 806 out_free_tags: 807 blk_mq_free_tag_set(&vblk->tag_set); 808 out_put_disk: 809 put_disk(vblk->disk); 810 out_free_vq: 811 vdev->config->del_vqs(vdev); 812 out_free_vblk: 813 kfree(vblk); 814 out_free_index: 815 ida_simple_remove(&vd_index_ida, index); 816 out: 817 return err; 818 } 819 820 static void virtblk_remove(struct virtio_device *vdev) 821 { 822 struct virtio_blk *vblk = vdev->priv; 823 int index = vblk->index; 824 int refc; 825 826 /* Make sure no work handler is accessing the device. */ 827 flush_work(&vblk->config_work); 828 829 del_gendisk(vblk->disk); 830 blk_cleanup_queue(vblk->disk->queue); 831 832 blk_mq_free_tag_set(&vblk->tag_set); 833 834 /* Stop all the virtqueues. */ 835 vdev->config->reset(vdev); 836 837 refc = kref_read(&disk_to_dev(vblk->disk)->kobj.kref); 838 put_disk(vblk->disk); 839 vdev->config->del_vqs(vdev); 840 kfree(vblk->vqs); 841 kfree(vblk); 842 843 /* Only free device id if we don't have any users */ 844 if (refc == 1) 845 ida_simple_remove(&vd_index_ida, index); 846 } 847 848 #ifdef CONFIG_PM_SLEEP 849 static int virtblk_freeze(struct virtio_device *vdev) 850 { 851 struct virtio_blk *vblk = vdev->priv; 852 853 /* Ensure we don't receive any more interrupts */ 854 vdev->config->reset(vdev); 855 856 /* Make sure no work handler is accessing the device. */ 857 flush_work(&vblk->config_work); 858 859 blk_mq_quiesce_queue(vblk->disk->queue); 860 861 vdev->config->del_vqs(vdev); 862 return 0; 863 } 864 865 static int virtblk_restore(struct virtio_device *vdev) 866 { 867 struct virtio_blk *vblk = vdev->priv; 868 int ret; 869 870 ret = init_vq(vdev->priv); 871 if (ret) 872 return ret; 873 874 virtio_device_ready(vdev); 875 876 blk_mq_unquiesce_queue(vblk->disk->queue); 877 return 0; 878 } 879 #endif 880 881 static const struct virtio_device_id id_table[] = { 882 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID }, 883 { 0 }, 884 }; 885 886 static unsigned int features_legacy[] = { 887 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY, 888 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, 889 VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, 890 VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES, 891 } 892 ; 893 static unsigned int features[] = { 894 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY, 895 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, 896 VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, 897 VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES, 898 }; 899 900 static struct virtio_driver virtio_blk = { 901 .feature_table = features, 902 .feature_table_size = ARRAY_SIZE(features), 903 .feature_table_legacy = features_legacy, 904 .feature_table_size_legacy = ARRAY_SIZE(features_legacy), 905 .driver.name = KBUILD_MODNAME, 906 .driver.owner = THIS_MODULE, 907 .id_table = id_table, 908 .probe = virtblk_probe, 909 .remove = virtblk_remove, 910 .config_changed = virtblk_config_changed, 911 #ifdef CONFIG_PM_SLEEP 912 .freeze = virtblk_freeze, 913 .restore = virtblk_restore, 914 #endif 915 }; 916 917 static int __init init(void) 918 { 919 int error; 920 921 virtblk_wq = alloc_workqueue("virtio-blk", 0, 0); 922 if (!virtblk_wq) 923 return -ENOMEM; 924 925 major = register_blkdev(0, "virtblk"); 926 if (major < 0) { 927 error = major; 928 goto out_destroy_workqueue; 929 } 930 931 error = register_virtio_driver(&virtio_blk); 932 if (error) 933 goto out_unregister_blkdev; 934 return 0; 935 936 out_unregister_blkdev: 937 unregister_blkdev(major, "virtblk"); 938 out_destroy_workqueue: 939 destroy_workqueue(virtblk_wq); 940 return error; 941 } 942 943 static void __exit fini(void) 944 { 945 unregister_virtio_driver(&virtio_blk); 946 unregister_blkdev(major, "virtblk"); 947 destroy_workqueue(virtblk_wq); 948 } 949 module_init(init); 950 module_exit(fini); 951 952 MODULE_DEVICE_TABLE(virtio, id_table); 953 MODULE_DESCRIPTION("Virtio block driver"); 954 MODULE_LICENSE("GPL"); 955