1 // SPDX-License-Identifier: GPL-2.0-only 2 //#define DEBUG 3 #include <linux/spinlock.h> 4 #include <linux/slab.h> 5 #include <linux/blkdev.h> 6 #include <linux/hdreg.h> 7 #include <linux/module.h> 8 #include <linux/mutex.h> 9 #include <linux/interrupt.h> 10 #include <linux/virtio.h> 11 #include <linux/virtio_blk.h> 12 #include <linux/scatterlist.h> 13 #include <linux/string_helpers.h> 14 #include <linux/idr.h> 15 #include <linux/blk-mq.h> 16 #include <linux/blk-mq-virtio.h> 17 #include <linux/numa.h> 18 19 #define PART_BITS 4 20 #define VQ_NAME_LEN 16 21 #define MAX_DISCARD_SEGMENTS 256u 22 23 static int major; 24 static DEFINE_IDA(vd_index_ida); 25 26 static struct workqueue_struct *virtblk_wq; 27 28 struct virtio_blk_vq { 29 struct virtqueue *vq; 30 spinlock_t lock; 31 char name[VQ_NAME_LEN]; 32 } ____cacheline_aligned_in_smp; 33 34 struct virtio_blk { 35 struct virtio_device *vdev; 36 37 /* The disk structure for the kernel. */ 38 struct gendisk *disk; 39 40 /* Block layer tags. */ 41 struct blk_mq_tag_set tag_set; 42 43 /* Process context for config space updates */ 44 struct work_struct config_work; 45 46 /* What host tells us, plus 2 for header & tailer. */ 47 unsigned int sg_elems; 48 49 /* Ida index - used to track minor number allocations. */ 50 int index; 51 52 /* num of vqs */ 53 int num_vqs; 54 struct virtio_blk_vq *vqs; 55 }; 56 57 struct virtblk_req { 58 struct virtio_blk_outhdr out_hdr; 59 u8 status; 60 struct scatterlist sg[]; 61 }; 62 63 static inline blk_status_t virtblk_result(struct virtblk_req *vbr) 64 { 65 switch (vbr->status) { 66 case VIRTIO_BLK_S_OK: 67 return BLK_STS_OK; 68 case VIRTIO_BLK_S_UNSUPP: 69 return BLK_STS_NOTSUPP; 70 default: 71 return BLK_STS_IOERR; 72 } 73 } 74 75 static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr, 76 struct scatterlist *data_sg, bool have_data) 77 { 78 struct scatterlist hdr, status, *sgs[3]; 79 unsigned int num_out = 0, num_in = 0; 80 81 sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr)); 82 sgs[num_out++] = &hdr; 83 84 if (have_data) { 85 if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT)) 86 sgs[num_out++] = data_sg; 87 else 88 sgs[num_out + num_in++] = data_sg; 89 } 90 91 sg_init_one(&status, &vbr->status, sizeof(vbr->status)); 92 sgs[num_out + num_in++] = &status; 93 94 return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC); 95 } 96 97 static int virtblk_setup_discard_write_zeroes(struct request *req, bool unmap) 98 { 99 unsigned short segments = blk_rq_nr_discard_segments(req); 100 unsigned short n = 0; 101 struct virtio_blk_discard_write_zeroes *range; 102 struct bio *bio; 103 u32 flags = 0; 104 105 if (unmap) 106 flags |= VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP; 107 108 range = kmalloc_array(segments, sizeof(*range), GFP_ATOMIC); 109 if (!range) 110 return -ENOMEM; 111 112 __rq_for_each_bio(bio, req) { 113 u64 sector = bio->bi_iter.bi_sector; 114 u32 num_sectors = bio->bi_iter.bi_size >> SECTOR_SHIFT; 115 116 range[n].flags = cpu_to_le32(flags); 117 range[n].num_sectors = cpu_to_le32(num_sectors); 118 range[n].sector = cpu_to_le64(sector); 119 n++; 120 } 121 122 req->special_vec.bv_page = virt_to_page(range); 123 req->special_vec.bv_offset = offset_in_page(range); 124 req->special_vec.bv_len = sizeof(*range) * segments; 125 req->rq_flags |= RQF_SPECIAL_PAYLOAD; 126 127 return 0; 128 } 129 130 static inline void virtblk_request_done(struct request *req) 131 { 132 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); 133 134 if (req->rq_flags & RQF_SPECIAL_PAYLOAD) { 135 kfree(page_address(req->special_vec.bv_page) + 136 req->special_vec.bv_offset); 137 } 138 139 blk_mq_end_request(req, virtblk_result(vbr)); 140 } 141 142 static void virtblk_done(struct virtqueue *vq) 143 { 144 struct virtio_blk *vblk = vq->vdev->priv; 145 bool req_done = false; 146 int qid = vq->index; 147 struct virtblk_req *vbr; 148 unsigned long flags; 149 unsigned int len; 150 151 spin_lock_irqsave(&vblk->vqs[qid].lock, flags); 152 do { 153 virtqueue_disable_cb(vq); 154 while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) { 155 struct request *req = blk_mq_rq_from_pdu(vbr); 156 157 blk_mq_complete_request(req); 158 req_done = true; 159 } 160 if (unlikely(virtqueue_is_broken(vq))) 161 break; 162 } while (!virtqueue_enable_cb(vq)); 163 164 /* In case queue is stopped waiting for more buffers. */ 165 if (req_done) 166 blk_mq_start_stopped_hw_queues(vblk->disk->queue, true); 167 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); 168 } 169 170 static void virtio_commit_rqs(struct blk_mq_hw_ctx *hctx) 171 { 172 struct virtio_blk *vblk = hctx->queue->queuedata; 173 struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num]; 174 bool kick; 175 176 spin_lock_irq(&vq->lock); 177 kick = virtqueue_kick_prepare(vq->vq); 178 spin_unlock_irq(&vq->lock); 179 180 if (kick) 181 virtqueue_notify(vq->vq); 182 } 183 184 static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx, 185 const struct blk_mq_queue_data *bd) 186 { 187 struct virtio_blk *vblk = hctx->queue->queuedata; 188 struct request *req = bd->rq; 189 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); 190 unsigned long flags; 191 unsigned int num; 192 int qid = hctx->queue_num; 193 int err; 194 bool notify = false; 195 bool unmap = false; 196 u32 type; 197 198 BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems); 199 200 switch (req_op(req)) { 201 case REQ_OP_READ: 202 case REQ_OP_WRITE: 203 type = 0; 204 break; 205 case REQ_OP_FLUSH: 206 type = VIRTIO_BLK_T_FLUSH; 207 break; 208 case REQ_OP_DISCARD: 209 type = VIRTIO_BLK_T_DISCARD; 210 break; 211 case REQ_OP_WRITE_ZEROES: 212 type = VIRTIO_BLK_T_WRITE_ZEROES; 213 unmap = !(req->cmd_flags & REQ_NOUNMAP); 214 break; 215 case REQ_OP_DRV_IN: 216 type = VIRTIO_BLK_T_GET_ID; 217 break; 218 default: 219 WARN_ON_ONCE(1); 220 return BLK_STS_IOERR; 221 } 222 223 vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, type); 224 vbr->out_hdr.sector = type ? 225 0 : cpu_to_virtio64(vblk->vdev, blk_rq_pos(req)); 226 vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(req)); 227 228 blk_mq_start_request(req); 229 230 if (type == VIRTIO_BLK_T_DISCARD || type == VIRTIO_BLK_T_WRITE_ZEROES) { 231 err = virtblk_setup_discard_write_zeroes(req, unmap); 232 if (err) 233 return BLK_STS_RESOURCE; 234 } 235 236 num = blk_rq_map_sg(hctx->queue, req, vbr->sg); 237 if (num) { 238 if (rq_data_dir(req) == WRITE) 239 vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_OUT); 240 else 241 vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_IN); 242 } 243 244 spin_lock_irqsave(&vblk->vqs[qid].lock, flags); 245 err = virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg, num); 246 if (err) { 247 virtqueue_kick(vblk->vqs[qid].vq); 248 /* Don't stop the queue if -ENOMEM: we may have failed to 249 * bounce the buffer due to global resource outage. 250 */ 251 if (err == -ENOSPC) 252 blk_mq_stop_hw_queue(hctx); 253 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); 254 switch (err) { 255 case -ENOSPC: 256 return BLK_STS_DEV_RESOURCE; 257 case -ENOMEM: 258 return BLK_STS_RESOURCE; 259 default: 260 return BLK_STS_IOERR; 261 } 262 } 263 264 if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq)) 265 notify = true; 266 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); 267 268 if (notify) 269 virtqueue_notify(vblk->vqs[qid].vq); 270 return BLK_STS_OK; 271 } 272 273 /* return id (s/n) string for *disk to *id_str 274 */ 275 static int virtblk_get_id(struct gendisk *disk, char *id_str) 276 { 277 struct virtio_blk *vblk = disk->private_data; 278 struct request_queue *q = vblk->disk->queue; 279 struct request *req; 280 int err; 281 282 req = blk_get_request(q, REQ_OP_DRV_IN, 0); 283 if (IS_ERR(req)) 284 return PTR_ERR(req); 285 286 err = blk_rq_map_kern(q, req, id_str, VIRTIO_BLK_ID_BYTES, GFP_KERNEL); 287 if (err) 288 goto out; 289 290 blk_execute_rq(vblk->disk->queue, vblk->disk, req, false); 291 err = blk_status_to_errno(virtblk_result(blk_mq_rq_to_pdu(req))); 292 out: 293 blk_put_request(req); 294 return err; 295 } 296 297 /* We provide getgeo only to please some old bootloader/partitioning tools */ 298 static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo) 299 { 300 struct virtio_blk *vblk = bd->bd_disk->private_data; 301 302 /* see if the host passed in geometry config */ 303 if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) { 304 virtio_cread(vblk->vdev, struct virtio_blk_config, 305 geometry.cylinders, &geo->cylinders); 306 virtio_cread(vblk->vdev, struct virtio_blk_config, 307 geometry.heads, &geo->heads); 308 virtio_cread(vblk->vdev, struct virtio_blk_config, 309 geometry.sectors, &geo->sectors); 310 } else { 311 /* some standard values, similar to sd */ 312 geo->heads = 1 << 6; 313 geo->sectors = 1 << 5; 314 geo->cylinders = get_capacity(bd->bd_disk) >> 11; 315 } 316 return 0; 317 } 318 319 static const struct block_device_operations virtblk_fops = { 320 .owner = THIS_MODULE, 321 .getgeo = virtblk_getgeo, 322 }; 323 324 static int index_to_minor(int index) 325 { 326 return index << PART_BITS; 327 } 328 329 static int minor_to_index(int minor) 330 { 331 return minor >> PART_BITS; 332 } 333 334 static ssize_t serial_show(struct device *dev, 335 struct device_attribute *attr, char *buf) 336 { 337 struct gendisk *disk = dev_to_disk(dev); 338 int err; 339 340 /* sysfs gives us a PAGE_SIZE buffer */ 341 BUILD_BUG_ON(PAGE_SIZE < VIRTIO_BLK_ID_BYTES); 342 343 buf[VIRTIO_BLK_ID_BYTES] = '\0'; 344 err = virtblk_get_id(disk, buf); 345 if (!err) 346 return strlen(buf); 347 348 if (err == -EIO) /* Unsupported? Make it empty. */ 349 return 0; 350 351 return err; 352 } 353 354 static DEVICE_ATTR_RO(serial); 355 356 /* The queue's logical block size must be set before calling this */ 357 static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize) 358 { 359 struct virtio_device *vdev = vblk->vdev; 360 struct request_queue *q = vblk->disk->queue; 361 char cap_str_2[10], cap_str_10[10]; 362 unsigned long long nblocks; 363 u64 capacity; 364 365 /* Host must always specify the capacity. */ 366 virtio_cread(vdev, struct virtio_blk_config, capacity, &capacity); 367 368 /* If capacity is too big, truncate with warning. */ 369 if ((sector_t)capacity != capacity) { 370 dev_warn(&vdev->dev, "Capacity %llu too large: truncating\n", 371 (unsigned long long)capacity); 372 capacity = (sector_t)-1; 373 } 374 375 nblocks = DIV_ROUND_UP_ULL(capacity, queue_logical_block_size(q) >> 9); 376 377 string_get_size(nblocks, queue_logical_block_size(q), 378 STRING_UNITS_2, cap_str_2, sizeof(cap_str_2)); 379 string_get_size(nblocks, queue_logical_block_size(q), 380 STRING_UNITS_10, cap_str_10, sizeof(cap_str_10)); 381 382 dev_notice(&vdev->dev, 383 "[%s] %s%llu %d-byte logical blocks (%s/%s)\n", 384 vblk->disk->disk_name, 385 resize ? "new size: " : "", 386 nblocks, 387 queue_logical_block_size(q), 388 cap_str_10, 389 cap_str_2); 390 391 set_capacity(vblk->disk, capacity); 392 } 393 394 static void virtblk_config_changed_work(struct work_struct *work) 395 { 396 struct virtio_blk *vblk = 397 container_of(work, struct virtio_blk, config_work); 398 char *envp[] = { "RESIZE=1", NULL }; 399 400 virtblk_update_capacity(vblk, true); 401 revalidate_disk(vblk->disk); 402 kobject_uevent_env(&disk_to_dev(vblk->disk)->kobj, KOBJ_CHANGE, envp); 403 } 404 405 static void virtblk_config_changed(struct virtio_device *vdev) 406 { 407 struct virtio_blk *vblk = vdev->priv; 408 409 queue_work(virtblk_wq, &vblk->config_work); 410 } 411 412 static int init_vq(struct virtio_blk *vblk) 413 { 414 int err; 415 int i; 416 vq_callback_t **callbacks; 417 const char **names; 418 struct virtqueue **vqs; 419 unsigned short num_vqs; 420 struct virtio_device *vdev = vblk->vdev; 421 struct irq_affinity desc = { 0, }; 422 423 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_MQ, 424 struct virtio_blk_config, num_queues, 425 &num_vqs); 426 if (err) 427 num_vqs = 1; 428 429 num_vqs = min_t(unsigned int, nr_cpu_ids, num_vqs); 430 431 vblk->vqs = kmalloc_array(num_vqs, sizeof(*vblk->vqs), GFP_KERNEL); 432 if (!vblk->vqs) 433 return -ENOMEM; 434 435 names = kmalloc_array(num_vqs, sizeof(*names), GFP_KERNEL); 436 callbacks = kmalloc_array(num_vqs, sizeof(*callbacks), GFP_KERNEL); 437 vqs = kmalloc_array(num_vqs, sizeof(*vqs), GFP_KERNEL); 438 if (!names || !callbacks || !vqs) { 439 err = -ENOMEM; 440 goto out; 441 } 442 443 for (i = 0; i < num_vqs; i++) { 444 callbacks[i] = virtblk_done; 445 snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%d", i); 446 names[i] = vblk->vqs[i].name; 447 } 448 449 /* Discover virtqueues and write information to configuration. */ 450 err = virtio_find_vqs(vdev, num_vqs, vqs, callbacks, names, &desc); 451 if (err) 452 goto out; 453 454 for (i = 0; i < num_vqs; i++) { 455 spin_lock_init(&vblk->vqs[i].lock); 456 vblk->vqs[i].vq = vqs[i]; 457 } 458 vblk->num_vqs = num_vqs; 459 460 out: 461 kfree(vqs); 462 kfree(callbacks); 463 kfree(names); 464 if (err) 465 kfree(vblk->vqs); 466 return err; 467 } 468 469 /* 470 * Legacy naming scheme used for virtio devices. We are stuck with it for 471 * virtio blk but don't ever use it for any new driver. 472 */ 473 static int virtblk_name_format(char *prefix, int index, char *buf, int buflen) 474 { 475 const int base = 'z' - 'a' + 1; 476 char *begin = buf + strlen(prefix); 477 char *end = buf + buflen; 478 char *p; 479 int unit; 480 481 p = end - 1; 482 *p = '\0'; 483 unit = base; 484 do { 485 if (p == begin) 486 return -EINVAL; 487 *--p = 'a' + (index % unit); 488 index = (index / unit) - 1; 489 } while (index >= 0); 490 491 memmove(begin, p, end - p); 492 memcpy(buf, prefix, strlen(prefix)); 493 494 return 0; 495 } 496 497 static int virtblk_get_cache_mode(struct virtio_device *vdev) 498 { 499 u8 writeback; 500 int err; 501 502 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE, 503 struct virtio_blk_config, wce, 504 &writeback); 505 506 /* 507 * If WCE is not configurable and flush is not available, 508 * assume no writeback cache is in use. 509 */ 510 if (err) 511 writeback = virtio_has_feature(vdev, VIRTIO_BLK_F_FLUSH); 512 513 return writeback; 514 } 515 516 static void virtblk_update_cache_mode(struct virtio_device *vdev) 517 { 518 u8 writeback = virtblk_get_cache_mode(vdev); 519 struct virtio_blk *vblk = vdev->priv; 520 521 blk_queue_write_cache(vblk->disk->queue, writeback, false); 522 revalidate_disk(vblk->disk); 523 } 524 525 static const char *const virtblk_cache_types[] = { 526 "write through", "write back" 527 }; 528 529 static ssize_t 530 cache_type_store(struct device *dev, struct device_attribute *attr, 531 const char *buf, size_t count) 532 { 533 struct gendisk *disk = dev_to_disk(dev); 534 struct virtio_blk *vblk = disk->private_data; 535 struct virtio_device *vdev = vblk->vdev; 536 int i; 537 538 BUG_ON(!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_CONFIG_WCE)); 539 i = sysfs_match_string(virtblk_cache_types, buf); 540 if (i < 0) 541 return i; 542 543 virtio_cwrite8(vdev, offsetof(struct virtio_blk_config, wce), i); 544 virtblk_update_cache_mode(vdev); 545 return count; 546 } 547 548 static ssize_t 549 cache_type_show(struct device *dev, struct device_attribute *attr, char *buf) 550 { 551 struct gendisk *disk = dev_to_disk(dev); 552 struct virtio_blk *vblk = disk->private_data; 553 u8 writeback = virtblk_get_cache_mode(vblk->vdev); 554 555 BUG_ON(writeback >= ARRAY_SIZE(virtblk_cache_types)); 556 return snprintf(buf, 40, "%s\n", virtblk_cache_types[writeback]); 557 } 558 559 static DEVICE_ATTR_RW(cache_type); 560 561 static struct attribute *virtblk_attrs[] = { 562 &dev_attr_serial.attr, 563 &dev_attr_cache_type.attr, 564 NULL, 565 }; 566 567 static umode_t virtblk_attrs_are_visible(struct kobject *kobj, 568 struct attribute *a, int n) 569 { 570 struct device *dev = container_of(kobj, struct device, kobj); 571 struct gendisk *disk = dev_to_disk(dev); 572 struct virtio_blk *vblk = disk->private_data; 573 struct virtio_device *vdev = vblk->vdev; 574 575 if (a == &dev_attr_cache_type.attr && 576 !virtio_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE)) 577 return S_IRUGO; 578 579 return a->mode; 580 } 581 582 static const struct attribute_group virtblk_attr_group = { 583 .attrs = virtblk_attrs, 584 .is_visible = virtblk_attrs_are_visible, 585 }; 586 587 static const struct attribute_group *virtblk_attr_groups[] = { 588 &virtblk_attr_group, 589 NULL, 590 }; 591 592 static int virtblk_init_request(struct blk_mq_tag_set *set, struct request *rq, 593 unsigned int hctx_idx, unsigned int numa_node) 594 { 595 struct virtio_blk *vblk = set->driver_data; 596 struct virtblk_req *vbr = blk_mq_rq_to_pdu(rq); 597 598 sg_init_table(vbr->sg, vblk->sg_elems); 599 return 0; 600 } 601 602 static int virtblk_map_queues(struct blk_mq_tag_set *set) 603 { 604 struct virtio_blk *vblk = set->driver_data; 605 606 return blk_mq_virtio_map_queues(&set->map[HCTX_TYPE_DEFAULT], 607 vblk->vdev, 0); 608 } 609 610 static const struct blk_mq_ops virtio_mq_ops = { 611 .queue_rq = virtio_queue_rq, 612 .commit_rqs = virtio_commit_rqs, 613 .complete = virtblk_request_done, 614 .init_request = virtblk_init_request, 615 .map_queues = virtblk_map_queues, 616 }; 617 618 static unsigned int virtblk_queue_depth; 619 module_param_named(queue_depth, virtblk_queue_depth, uint, 0444); 620 621 static int virtblk_probe(struct virtio_device *vdev) 622 { 623 struct virtio_blk *vblk; 624 struct request_queue *q; 625 int err, index; 626 627 u32 v, blk_size, max_size, sg_elems, opt_io_size; 628 u16 min_io_size; 629 u8 physical_block_exp, alignment_offset; 630 631 if (!vdev->config->get) { 632 dev_err(&vdev->dev, "%s failure: config access disabled\n", 633 __func__); 634 return -EINVAL; 635 } 636 637 err = ida_simple_get(&vd_index_ida, 0, minor_to_index(1 << MINORBITS), 638 GFP_KERNEL); 639 if (err < 0) 640 goto out; 641 index = err; 642 643 /* We need to know how many segments before we allocate. */ 644 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SEG_MAX, 645 struct virtio_blk_config, seg_max, 646 &sg_elems); 647 648 /* We need at least one SG element, whatever they say. */ 649 if (err || !sg_elems) 650 sg_elems = 1; 651 652 /* We need an extra sg elements at head and tail. */ 653 sg_elems += 2; 654 vdev->priv = vblk = kmalloc(sizeof(*vblk), GFP_KERNEL); 655 if (!vblk) { 656 err = -ENOMEM; 657 goto out_free_index; 658 } 659 660 vblk->vdev = vdev; 661 vblk->sg_elems = sg_elems; 662 663 INIT_WORK(&vblk->config_work, virtblk_config_changed_work); 664 665 err = init_vq(vblk); 666 if (err) 667 goto out_free_vblk; 668 669 /* FIXME: How many partitions? How long is a piece of string? */ 670 vblk->disk = alloc_disk(1 << PART_BITS); 671 if (!vblk->disk) { 672 err = -ENOMEM; 673 goto out_free_vq; 674 } 675 676 /* Default queue sizing is to fill the ring. */ 677 if (!virtblk_queue_depth) { 678 virtblk_queue_depth = vblk->vqs[0].vq->num_free; 679 /* ... but without indirect descs, we use 2 descs per req */ 680 if (!virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) 681 virtblk_queue_depth /= 2; 682 } 683 684 memset(&vblk->tag_set, 0, sizeof(vblk->tag_set)); 685 vblk->tag_set.ops = &virtio_mq_ops; 686 vblk->tag_set.queue_depth = virtblk_queue_depth; 687 vblk->tag_set.numa_node = NUMA_NO_NODE; 688 vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; 689 vblk->tag_set.cmd_size = 690 sizeof(struct virtblk_req) + 691 sizeof(struct scatterlist) * sg_elems; 692 vblk->tag_set.driver_data = vblk; 693 vblk->tag_set.nr_hw_queues = vblk->num_vqs; 694 695 err = blk_mq_alloc_tag_set(&vblk->tag_set); 696 if (err) 697 goto out_put_disk; 698 699 q = blk_mq_init_queue(&vblk->tag_set); 700 if (IS_ERR(q)) { 701 err = -ENOMEM; 702 goto out_free_tags; 703 } 704 vblk->disk->queue = q; 705 706 q->queuedata = vblk; 707 708 virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN); 709 710 vblk->disk->major = major; 711 vblk->disk->first_minor = index_to_minor(index); 712 vblk->disk->private_data = vblk; 713 vblk->disk->fops = &virtblk_fops; 714 vblk->disk->flags |= GENHD_FL_EXT_DEVT; 715 vblk->index = index; 716 717 /* configure queue flush support */ 718 virtblk_update_cache_mode(vdev); 719 720 /* If disk is read-only in the host, the guest should obey */ 721 if (virtio_has_feature(vdev, VIRTIO_BLK_F_RO)) 722 set_disk_ro(vblk->disk, 1); 723 724 /* We can handle whatever the host told us to handle. */ 725 blk_queue_max_segments(q, vblk->sg_elems-2); 726 727 /* No real sector limit. */ 728 blk_queue_max_hw_sectors(q, -1U); 729 730 max_size = virtio_max_dma_size(vdev); 731 732 /* Host can optionally specify maximum segment size and number of 733 * segments. */ 734 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SIZE_MAX, 735 struct virtio_blk_config, size_max, &v); 736 if (!err) 737 max_size = min(max_size, v); 738 739 blk_queue_max_segment_size(q, max_size); 740 741 /* Host can optionally specify the block size of the device */ 742 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_BLK_SIZE, 743 struct virtio_blk_config, blk_size, 744 &blk_size); 745 if (!err) 746 blk_queue_logical_block_size(q, blk_size); 747 else 748 blk_size = queue_logical_block_size(q); 749 750 /* Use topology information if available */ 751 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 752 struct virtio_blk_config, physical_block_exp, 753 &physical_block_exp); 754 if (!err && physical_block_exp) 755 blk_queue_physical_block_size(q, 756 blk_size * (1 << physical_block_exp)); 757 758 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 759 struct virtio_blk_config, alignment_offset, 760 &alignment_offset); 761 if (!err && alignment_offset) 762 blk_queue_alignment_offset(q, blk_size * alignment_offset); 763 764 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 765 struct virtio_blk_config, min_io_size, 766 &min_io_size); 767 if (!err && min_io_size) 768 blk_queue_io_min(q, blk_size * min_io_size); 769 770 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 771 struct virtio_blk_config, opt_io_size, 772 &opt_io_size); 773 if (!err && opt_io_size) 774 blk_queue_io_opt(q, blk_size * opt_io_size); 775 776 if (virtio_has_feature(vdev, VIRTIO_BLK_F_DISCARD)) { 777 q->limits.discard_granularity = blk_size; 778 779 virtio_cread(vdev, struct virtio_blk_config, 780 discard_sector_alignment, &v); 781 q->limits.discard_alignment = v ? v << SECTOR_SHIFT : 0; 782 783 virtio_cread(vdev, struct virtio_blk_config, 784 max_discard_sectors, &v); 785 blk_queue_max_discard_sectors(q, v ? v : UINT_MAX); 786 787 virtio_cread(vdev, struct virtio_blk_config, max_discard_seg, 788 &v); 789 blk_queue_max_discard_segments(q, 790 min_not_zero(v, 791 MAX_DISCARD_SEGMENTS)); 792 793 blk_queue_flag_set(QUEUE_FLAG_DISCARD, q); 794 } 795 796 if (virtio_has_feature(vdev, VIRTIO_BLK_F_WRITE_ZEROES)) { 797 virtio_cread(vdev, struct virtio_blk_config, 798 max_write_zeroes_sectors, &v); 799 blk_queue_max_write_zeroes_sectors(q, v ? v : UINT_MAX); 800 } 801 802 virtblk_update_capacity(vblk, false); 803 virtio_device_ready(vdev); 804 805 device_add_disk(&vdev->dev, vblk->disk, virtblk_attr_groups); 806 return 0; 807 808 out_free_tags: 809 blk_mq_free_tag_set(&vblk->tag_set); 810 out_put_disk: 811 put_disk(vblk->disk); 812 out_free_vq: 813 vdev->config->del_vqs(vdev); 814 out_free_vblk: 815 kfree(vblk); 816 out_free_index: 817 ida_simple_remove(&vd_index_ida, index); 818 out: 819 return err; 820 } 821 822 static void virtblk_remove(struct virtio_device *vdev) 823 { 824 struct virtio_blk *vblk = vdev->priv; 825 int index = vblk->index; 826 int refc; 827 828 /* Make sure no work handler is accessing the device. */ 829 flush_work(&vblk->config_work); 830 831 del_gendisk(vblk->disk); 832 blk_cleanup_queue(vblk->disk->queue); 833 834 blk_mq_free_tag_set(&vblk->tag_set); 835 836 /* Stop all the virtqueues. */ 837 vdev->config->reset(vdev); 838 839 refc = kref_read(&disk_to_dev(vblk->disk)->kobj.kref); 840 put_disk(vblk->disk); 841 vdev->config->del_vqs(vdev); 842 kfree(vblk->vqs); 843 kfree(vblk); 844 845 /* Only free device id if we don't have any users */ 846 if (refc == 1) 847 ida_simple_remove(&vd_index_ida, index); 848 } 849 850 #ifdef CONFIG_PM_SLEEP 851 static int virtblk_freeze(struct virtio_device *vdev) 852 { 853 struct virtio_blk *vblk = vdev->priv; 854 855 /* Ensure we don't receive any more interrupts */ 856 vdev->config->reset(vdev); 857 858 /* Make sure no work handler is accessing the device. */ 859 flush_work(&vblk->config_work); 860 861 blk_mq_quiesce_queue(vblk->disk->queue); 862 863 vdev->config->del_vqs(vdev); 864 return 0; 865 } 866 867 static int virtblk_restore(struct virtio_device *vdev) 868 { 869 struct virtio_blk *vblk = vdev->priv; 870 int ret; 871 872 ret = init_vq(vdev->priv); 873 if (ret) 874 return ret; 875 876 virtio_device_ready(vdev); 877 878 blk_mq_unquiesce_queue(vblk->disk->queue); 879 return 0; 880 } 881 #endif 882 883 static const struct virtio_device_id id_table[] = { 884 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID }, 885 { 0 }, 886 }; 887 888 static unsigned int features_legacy[] = { 889 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY, 890 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, 891 VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, 892 VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES, 893 } 894 ; 895 static unsigned int features[] = { 896 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY, 897 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, 898 VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, 899 VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES, 900 }; 901 902 static struct virtio_driver virtio_blk = { 903 .feature_table = features, 904 .feature_table_size = ARRAY_SIZE(features), 905 .feature_table_legacy = features_legacy, 906 .feature_table_size_legacy = ARRAY_SIZE(features_legacy), 907 .driver.name = KBUILD_MODNAME, 908 .driver.owner = THIS_MODULE, 909 .id_table = id_table, 910 .probe = virtblk_probe, 911 .remove = virtblk_remove, 912 .config_changed = virtblk_config_changed, 913 #ifdef CONFIG_PM_SLEEP 914 .freeze = virtblk_freeze, 915 .restore = virtblk_restore, 916 #endif 917 }; 918 919 static int __init init(void) 920 { 921 int error; 922 923 virtblk_wq = alloc_workqueue("virtio-blk", 0, 0); 924 if (!virtblk_wq) 925 return -ENOMEM; 926 927 major = register_blkdev(0, "virtblk"); 928 if (major < 0) { 929 error = major; 930 goto out_destroy_workqueue; 931 } 932 933 error = register_virtio_driver(&virtio_blk); 934 if (error) 935 goto out_unregister_blkdev; 936 return 0; 937 938 out_unregister_blkdev: 939 unregister_blkdev(major, "virtblk"); 940 out_destroy_workqueue: 941 destroy_workqueue(virtblk_wq); 942 return error; 943 } 944 945 static void __exit fini(void) 946 { 947 unregister_virtio_driver(&virtio_blk); 948 unregister_blkdev(major, "virtblk"); 949 destroy_workqueue(virtblk_wq); 950 } 951 module_init(init); 952 module_exit(fini); 953 954 MODULE_DEVICE_TABLE(virtio, id_table); 955 MODULE_DESCRIPTION("Virtio block driver"); 956 MODULE_LICENSE("GPL"); 957