1 // SPDX-License-Identifier: GPL-2.0-only 2 //#define DEBUG 3 #include <linux/spinlock.h> 4 #include <linux/slab.h> 5 #include <linux/blkdev.h> 6 #include <linux/hdreg.h> 7 #include <linux/module.h> 8 #include <linux/mutex.h> 9 #include <linux/interrupt.h> 10 #include <linux/virtio.h> 11 #include <linux/virtio_blk.h> 12 #include <linux/scatterlist.h> 13 #include <linux/string_helpers.h> 14 #include <linux/idr.h> 15 #include <linux/blk-mq.h> 16 #include <linux/blk-mq-virtio.h> 17 #include <linux/numa.h> 18 #include <uapi/linux/virtio_ring.h> 19 20 #define PART_BITS 4 21 #define VQ_NAME_LEN 16 22 #define MAX_DISCARD_SEGMENTS 256u 23 24 /* The maximum number of sg elements that fit into a virtqueue */ 25 #define VIRTIO_BLK_MAX_SG_ELEMS 32768 26 27 #ifdef CONFIG_ARCH_NO_SG_CHAIN 28 #define VIRTIO_BLK_INLINE_SG_CNT 0 29 #else 30 #define VIRTIO_BLK_INLINE_SG_CNT 2 31 #endif 32 33 static unsigned int num_request_queues; 34 module_param(num_request_queues, uint, 0644); 35 MODULE_PARM_DESC(num_request_queues, 36 "Limit the number of request queues to use for blk device. " 37 "0 for no limit. " 38 "Values > nr_cpu_ids truncated to nr_cpu_ids."); 39 40 static unsigned int poll_queues; 41 module_param(poll_queues, uint, 0644); 42 MODULE_PARM_DESC(poll_queues, "The number of dedicated virtqueues for polling I/O"); 43 44 static int major; 45 static DEFINE_IDA(vd_index_ida); 46 47 static struct workqueue_struct *virtblk_wq; 48 49 struct virtio_blk_vq { 50 struct virtqueue *vq; 51 spinlock_t lock; 52 char name[VQ_NAME_LEN]; 53 } ____cacheline_aligned_in_smp; 54 55 struct virtio_blk { 56 /* 57 * This mutex must be held by anything that may run after 58 * virtblk_remove() sets vblk->vdev to NULL. 59 * 60 * blk-mq, virtqueue processing, and sysfs attribute code paths are 61 * shut down before vblk->vdev is set to NULL and therefore do not need 62 * to hold this mutex. 63 */ 64 struct mutex vdev_mutex; 65 struct virtio_device *vdev; 66 67 /* The disk structure for the kernel. */ 68 struct gendisk *disk; 69 70 /* Block layer tags. */ 71 struct blk_mq_tag_set tag_set; 72 73 /* Process context for config space updates */ 74 struct work_struct config_work; 75 76 /* Ida index - used to track minor number allocations. */ 77 int index; 78 79 /* num of vqs */ 80 int num_vqs; 81 int io_queues[HCTX_MAX_TYPES]; 82 struct virtio_blk_vq *vqs; 83 }; 84 85 struct virtblk_req { 86 struct virtio_blk_outhdr out_hdr; 87 u8 status; 88 struct sg_table sg_table; 89 struct scatterlist sg[]; 90 }; 91 92 static inline blk_status_t virtblk_result(struct virtblk_req *vbr) 93 { 94 switch (vbr->status) { 95 case VIRTIO_BLK_S_OK: 96 return BLK_STS_OK; 97 case VIRTIO_BLK_S_UNSUPP: 98 return BLK_STS_NOTSUPP; 99 default: 100 return BLK_STS_IOERR; 101 } 102 } 103 104 static inline struct virtio_blk_vq *get_virtio_blk_vq(struct blk_mq_hw_ctx *hctx) 105 { 106 struct virtio_blk *vblk = hctx->queue->queuedata; 107 struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num]; 108 109 return vq; 110 } 111 112 static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr) 113 { 114 struct scatterlist hdr, status, *sgs[3]; 115 unsigned int num_out = 0, num_in = 0; 116 117 sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr)); 118 sgs[num_out++] = &hdr; 119 120 if (vbr->sg_table.nents) { 121 if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT)) 122 sgs[num_out++] = vbr->sg_table.sgl; 123 else 124 sgs[num_out + num_in++] = vbr->sg_table.sgl; 125 } 126 127 sg_init_one(&status, &vbr->status, sizeof(vbr->status)); 128 sgs[num_out + num_in++] = &status; 129 130 return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC); 131 } 132 133 static int virtblk_setup_discard_write_zeroes(struct request *req, bool unmap) 134 { 135 unsigned short segments = blk_rq_nr_discard_segments(req); 136 unsigned short n = 0; 137 struct virtio_blk_discard_write_zeroes *range; 138 struct bio *bio; 139 u32 flags = 0; 140 141 if (unmap) 142 flags |= VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP; 143 144 range = kmalloc_array(segments, sizeof(*range), GFP_ATOMIC); 145 if (!range) 146 return -ENOMEM; 147 148 /* 149 * Single max discard segment means multi-range discard isn't 150 * supported, and block layer only runs contiguity merge like 151 * normal RW request. So we can't reply on bio for retrieving 152 * each range info. 153 */ 154 if (queue_max_discard_segments(req->q) == 1) { 155 range[0].flags = cpu_to_le32(flags); 156 range[0].num_sectors = cpu_to_le32(blk_rq_sectors(req)); 157 range[0].sector = cpu_to_le64(blk_rq_pos(req)); 158 n = 1; 159 } else { 160 __rq_for_each_bio(bio, req) { 161 u64 sector = bio->bi_iter.bi_sector; 162 u32 num_sectors = bio->bi_iter.bi_size >> SECTOR_SHIFT; 163 164 range[n].flags = cpu_to_le32(flags); 165 range[n].num_sectors = cpu_to_le32(num_sectors); 166 range[n].sector = cpu_to_le64(sector); 167 n++; 168 } 169 } 170 171 WARN_ON_ONCE(n != segments); 172 173 req->special_vec.bv_page = virt_to_page(range); 174 req->special_vec.bv_offset = offset_in_page(range); 175 req->special_vec.bv_len = sizeof(*range) * segments; 176 req->rq_flags |= RQF_SPECIAL_PAYLOAD; 177 178 return 0; 179 } 180 181 static void virtblk_unmap_data(struct request *req, struct virtblk_req *vbr) 182 { 183 if (blk_rq_nr_phys_segments(req)) 184 sg_free_table_chained(&vbr->sg_table, 185 VIRTIO_BLK_INLINE_SG_CNT); 186 } 187 188 static int virtblk_map_data(struct blk_mq_hw_ctx *hctx, struct request *req, 189 struct virtblk_req *vbr) 190 { 191 int err; 192 193 if (!blk_rq_nr_phys_segments(req)) 194 return 0; 195 196 vbr->sg_table.sgl = vbr->sg; 197 err = sg_alloc_table_chained(&vbr->sg_table, 198 blk_rq_nr_phys_segments(req), 199 vbr->sg_table.sgl, 200 VIRTIO_BLK_INLINE_SG_CNT); 201 if (unlikely(err)) 202 return -ENOMEM; 203 204 return blk_rq_map_sg(hctx->queue, req, vbr->sg_table.sgl); 205 } 206 207 static void virtblk_cleanup_cmd(struct request *req) 208 { 209 if (req->rq_flags & RQF_SPECIAL_PAYLOAD) 210 kfree(bvec_virt(&req->special_vec)); 211 } 212 213 static blk_status_t virtblk_setup_cmd(struct virtio_device *vdev, 214 struct request *req, 215 struct virtblk_req *vbr) 216 { 217 bool unmap = false; 218 u32 type; 219 220 vbr->out_hdr.sector = 0; 221 222 switch (req_op(req)) { 223 case REQ_OP_READ: 224 type = VIRTIO_BLK_T_IN; 225 vbr->out_hdr.sector = cpu_to_virtio64(vdev, 226 blk_rq_pos(req)); 227 break; 228 case REQ_OP_WRITE: 229 type = VIRTIO_BLK_T_OUT; 230 vbr->out_hdr.sector = cpu_to_virtio64(vdev, 231 blk_rq_pos(req)); 232 break; 233 case REQ_OP_FLUSH: 234 type = VIRTIO_BLK_T_FLUSH; 235 break; 236 case REQ_OP_DISCARD: 237 type = VIRTIO_BLK_T_DISCARD; 238 break; 239 case REQ_OP_WRITE_ZEROES: 240 type = VIRTIO_BLK_T_WRITE_ZEROES; 241 unmap = !(req->cmd_flags & REQ_NOUNMAP); 242 break; 243 case REQ_OP_DRV_IN: 244 type = VIRTIO_BLK_T_GET_ID; 245 break; 246 default: 247 WARN_ON_ONCE(1); 248 return BLK_STS_IOERR; 249 } 250 251 vbr->out_hdr.type = cpu_to_virtio32(vdev, type); 252 vbr->out_hdr.ioprio = cpu_to_virtio32(vdev, req_get_ioprio(req)); 253 254 if (type == VIRTIO_BLK_T_DISCARD || type == VIRTIO_BLK_T_WRITE_ZEROES) { 255 if (virtblk_setup_discard_write_zeroes(req, unmap)) 256 return BLK_STS_RESOURCE; 257 } 258 259 return 0; 260 } 261 262 static inline void virtblk_request_done(struct request *req) 263 { 264 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); 265 266 virtblk_unmap_data(req, vbr); 267 virtblk_cleanup_cmd(req); 268 blk_mq_end_request(req, virtblk_result(vbr)); 269 } 270 271 static void virtblk_done(struct virtqueue *vq) 272 { 273 struct virtio_blk *vblk = vq->vdev->priv; 274 bool req_done = false; 275 int qid = vq->index; 276 struct virtblk_req *vbr; 277 unsigned long flags; 278 unsigned int len; 279 280 spin_lock_irqsave(&vblk->vqs[qid].lock, flags); 281 do { 282 virtqueue_disable_cb(vq); 283 while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) { 284 struct request *req = blk_mq_rq_from_pdu(vbr); 285 286 if (likely(!blk_should_fake_timeout(req->q))) 287 blk_mq_complete_request(req); 288 req_done = true; 289 } 290 if (unlikely(virtqueue_is_broken(vq))) 291 break; 292 } while (!virtqueue_enable_cb(vq)); 293 294 /* In case queue is stopped waiting for more buffers. */ 295 if (req_done) 296 blk_mq_start_stopped_hw_queues(vblk->disk->queue, true); 297 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); 298 } 299 300 static void virtio_commit_rqs(struct blk_mq_hw_ctx *hctx) 301 { 302 struct virtio_blk *vblk = hctx->queue->queuedata; 303 struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num]; 304 bool kick; 305 306 spin_lock_irq(&vq->lock); 307 kick = virtqueue_kick_prepare(vq->vq); 308 spin_unlock_irq(&vq->lock); 309 310 if (kick) 311 virtqueue_notify(vq->vq); 312 } 313 314 static blk_status_t virtblk_prep_rq(struct blk_mq_hw_ctx *hctx, 315 struct virtio_blk *vblk, 316 struct request *req, 317 struct virtblk_req *vbr) 318 { 319 blk_status_t status; 320 321 status = virtblk_setup_cmd(vblk->vdev, req, vbr); 322 if (unlikely(status)) 323 return status; 324 325 vbr->sg_table.nents = virtblk_map_data(hctx, req, vbr); 326 if (unlikely(vbr->sg_table.nents < 0)) { 327 virtblk_cleanup_cmd(req); 328 return BLK_STS_RESOURCE; 329 } 330 331 blk_mq_start_request(req); 332 333 return BLK_STS_OK; 334 } 335 336 static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx, 337 const struct blk_mq_queue_data *bd) 338 { 339 struct virtio_blk *vblk = hctx->queue->queuedata; 340 struct request *req = bd->rq; 341 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); 342 unsigned long flags; 343 int qid = hctx->queue_num; 344 bool notify = false; 345 blk_status_t status; 346 int err; 347 348 status = virtblk_prep_rq(hctx, vblk, req, vbr); 349 if (unlikely(status)) 350 return status; 351 352 spin_lock_irqsave(&vblk->vqs[qid].lock, flags); 353 err = virtblk_add_req(vblk->vqs[qid].vq, vbr); 354 if (err) { 355 virtqueue_kick(vblk->vqs[qid].vq); 356 /* Don't stop the queue if -ENOMEM: we may have failed to 357 * bounce the buffer due to global resource outage. 358 */ 359 if (err == -ENOSPC) 360 blk_mq_stop_hw_queue(hctx); 361 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); 362 virtblk_unmap_data(req, vbr); 363 virtblk_cleanup_cmd(req); 364 switch (err) { 365 case -ENOSPC: 366 return BLK_STS_DEV_RESOURCE; 367 case -ENOMEM: 368 return BLK_STS_RESOURCE; 369 default: 370 return BLK_STS_IOERR; 371 } 372 } 373 374 if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq)) 375 notify = true; 376 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); 377 378 if (notify) 379 virtqueue_notify(vblk->vqs[qid].vq); 380 return BLK_STS_OK; 381 } 382 383 static bool virtblk_prep_rq_batch(struct request *req) 384 { 385 struct virtio_blk *vblk = req->mq_hctx->queue->queuedata; 386 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); 387 388 req->mq_hctx->tags->rqs[req->tag] = req; 389 390 return virtblk_prep_rq(req->mq_hctx, vblk, req, vbr) == BLK_STS_OK; 391 } 392 393 static bool virtblk_add_req_batch(struct virtio_blk_vq *vq, 394 struct request **rqlist) 395 { 396 unsigned long flags; 397 int err; 398 bool kick; 399 400 spin_lock_irqsave(&vq->lock, flags); 401 402 while (!rq_list_empty(*rqlist)) { 403 struct request *req = rq_list_pop(rqlist); 404 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); 405 406 err = virtblk_add_req(vq->vq, vbr); 407 if (err) { 408 virtblk_unmap_data(req, vbr); 409 virtblk_cleanup_cmd(req); 410 blk_mq_requeue_request(req, true); 411 } 412 } 413 414 kick = virtqueue_kick_prepare(vq->vq); 415 spin_unlock_irqrestore(&vq->lock, flags); 416 417 return kick; 418 } 419 420 static void virtio_queue_rqs(struct request **rqlist) 421 { 422 struct request *req, *next, *prev = NULL; 423 struct request *requeue_list = NULL; 424 425 rq_list_for_each_safe(rqlist, req, next) { 426 struct virtio_blk_vq *vq = get_virtio_blk_vq(req->mq_hctx); 427 bool kick; 428 429 if (!virtblk_prep_rq_batch(req)) { 430 rq_list_move(rqlist, &requeue_list, req, prev); 431 req = prev; 432 if (!req) 433 continue; 434 } 435 436 if (!next || req->mq_hctx != next->mq_hctx) { 437 req->rq_next = NULL; 438 kick = virtblk_add_req_batch(vq, rqlist); 439 if (kick) 440 virtqueue_notify(vq->vq); 441 442 *rqlist = next; 443 prev = NULL; 444 } else 445 prev = req; 446 } 447 448 *rqlist = requeue_list; 449 } 450 451 /* return id (s/n) string for *disk to *id_str 452 */ 453 static int virtblk_get_id(struct gendisk *disk, char *id_str) 454 { 455 struct virtio_blk *vblk = disk->private_data; 456 struct request_queue *q = vblk->disk->queue; 457 struct request *req; 458 int err; 459 460 req = blk_mq_alloc_request(q, REQ_OP_DRV_IN, 0); 461 if (IS_ERR(req)) 462 return PTR_ERR(req); 463 464 err = blk_rq_map_kern(q, req, id_str, VIRTIO_BLK_ID_BYTES, GFP_KERNEL); 465 if (err) 466 goto out; 467 468 blk_execute_rq(req, false); 469 err = blk_status_to_errno(virtblk_result(blk_mq_rq_to_pdu(req))); 470 out: 471 blk_mq_free_request(req); 472 return err; 473 } 474 475 /* We provide getgeo only to please some old bootloader/partitioning tools */ 476 static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo) 477 { 478 struct virtio_blk *vblk = bd->bd_disk->private_data; 479 int ret = 0; 480 481 mutex_lock(&vblk->vdev_mutex); 482 483 if (!vblk->vdev) { 484 ret = -ENXIO; 485 goto out; 486 } 487 488 /* see if the host passed in geometry config */ 489 if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) { 490 virtio_cread(vblk->vdev, struct virtio_blk_config, 491 geometry.cylinders, &geo->cylinders); 492 virtio_cread(vblk->vdev, struct virtio_blk_config, 493 geometry.heads, &geo->heads); 494 virtio_cread(vblk->vdev, struct virtio_blk_config, 495 geometry.sectors, &geo->sectors); 496 } else { 497 /* some standard values, similar to sd */ 498 geo->heads = 1 << 6; 499 geo->sectors = 1 << 5; 500 geo->cylinders = get_capacity(bd->bd_disk) >> 11; 501 } 502 out: 503 mutex_unlock(&vblk->vdev_mutex); 504 return ret; 505 } 506 507 static void virtblk_free_disk(struct gendisk *disk) 508 { 509 struct virtio_blk *vblk = disk->private_data; 510 511 ida_simple_remove(&vd_index_ida, vblk->index); 512 mutex_destroy(&vblk->vdev_mutex); 513 kfree(vblk); 514 } 515 516 static const struct block_device_operations virtblk_fops = { 517 .owner = THIS_MODULE, 518 .getgeo = virtblk_getgeo, 519 .free_disk = virtblk_free_disk, 520 }; 521 522 static int index_to_minor(int index) 523 { 524 return index << PART_BITS; 525 } 526 527 static int minor_to_index(int minor) 528 { 529 return minor >> PART_BITS; 530 } 531 532 static ssize_t serial_show(struct device *dev, 533 struct device_attribute *attr, char *buf) 534 { 535 struct gendisk *disk = dev_to_disk(dev); 536 int err; 537 538 /* sysfs gives us a PAGE_SIZE buffer */ 539 BUILD_BUG_ON(PAGE_SIZE < VIRTIO_BLK_ID_BYTES); 540 541 buf[VIRTIO_BLK_ID_BYTES] = '\0'; 542 err = virtblk_get_id(disk, buf); 543 if (!err) 544 return strlen(buf); 545 546 if (err == -EIO) /* Unsupported? Make it empty. */ 547 return 0; 548 549 return err; 550 } 551 552 static DEVICE_ATTR_RO(serial); 553 554 /* The queue's logical block size must be set before calling this */ 555 static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize) 556 { 557 struct virtio_device *vdev = vblk->vdev; 558 struct request_queue *q = vblk->disk->queue; 559 char cap_str_2[10], cap_str_10[10]; 560 unsigned long long nblocks; 561 u64 capacity; 562 563 /* Host must always specify the capacity. */ 564 virtio_cread(vdev, struct virtio_blk_config, capacity, &capacity); 565 566 nblocks = DIV_ROUND_UP_ULL(capacity, queue_logical_block_size(q) >> 9); 567 568 string_get_size(nblocks, queue_logical_block_size(q), 569 STRING_UNITS_2, cap_str_2, sizeof(cap_str_2)); 570 string_get_size(nblocks, queue_logical_block_size(q), 571 STRING_UNITS_10, cap_str_10, sizeof(cap_str_10)); 572 573 dev_notice(&vdev->dev, 574 "[%s] %s%llu %d-byte logical blocks (%s/%s)\n", 575 vblk->disk->disk_name, 576 resize ? "new size: " : "", 577 nblocks, 578 queue_logical_block_size(q), 579 cap_str_10, 580 cap_str_2); 581 582 set_capacity_and_notify(vblk->disk, capacity); 583 } 584 585 static void virtblk_config_changed_work(struct work_struct *work) 586 { 587 struct virtio_blk *vblk = 588 container_of(work, struct virtio_blk, config_work); 589 590 virtblk_update_capacity(vblk, true); 591 } 592 593 static void virtblk_config_changed(struct virtio_device *vdev) 594 { 595 struct virtio_blk *vblk = vdev->priv; 596 597 queue_work(virtblk_wq, &vblk->config_work); 598 } 599 600 static int init_vq(struct virtio_blk *vblk) 601 { 602 int err; 603 int i; 604 vq_callback_t **callbacks; 605 const char **names; 606 struct virtqueue **vqs; 607 unsigned short num_vqs; 608 unsigned int num_poll_vqs; 609 struct virtio_device *vdev = vblk->vdev; 610 struct irq_affinity desc = { 0, }; 611 612 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_MQ, 613 struct virtio_blk_config, num_queues, 614 &num_vqs); 615 if (err) 616 num_vqs = 1; 617 618 if (!err && !num_vqs) { 619 dev_err(&vdev->dev, "MQ advertised but zero queues reported\n"); 620 return -EINVAL; 621 } 622 623 num_vqs = min_t(unsigned int, 624 min_not_zero(num_request_queues, nr_cpu_ids), 625 num_vqs); 626 627 num_poll_vqs = min_t(unsigned int, poll_queues, num_vqs - 1); 628 629 vblk->io_queues[HCTX_TYPE_DEFAULT] = num_vqs - num_poll_vqs; 630 vblk->io_queues[HCTX_TYPE_READ] = 0; 631 vblk->io_queues[HCTX_TYPE_POLL] = num_poll_vqs; 632 633 dev_info(&vdev->dev, "%d/%d/%d default/read/poll queues\n", 634 vblk->io_queues[HCTX_TYPE_DEFAULT], 635 vblk->io_queues[HCTX_TYPE_READ], 636 vblk->io_queues[HCTX_TYPE_POLL]); 637 638 vblk->vqs = kmalloc_array(num_vqs, sizeof(*vblk->vqs), GFP_KERNEL); 639 if (!vblk->vqs) 640 return -ENOMEM; 641 642 names = kmalloc_array(num_vqs, sizeof(*names), GFP_KERNEL); 643 callbacks = kmalloc_array(num_vqs, sizeof(*callbacks), GFP_KERNEL); 644 vqs = kmalloc_array(num_vqs, sizeof(*vqs), GFP_KERNEL); 645 if (!names || !callbacks || !vqs) { 646 err = -ENOMEM; 647 goto out; 648 } 649 650 for (i = 0; i < num_vqs - num_poll_vqs; i++) { 651 callbacks[i] = virtblk_done; 652 snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%d", i); 653 names[i] = vblk->vqs[i].name; 654 } 655 656 for (; i < num_vqs; i++) { 657 callbacks[i] = NULL; 658 snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req_poll.%d", i); 659 names[i] = vblk->vqs[i].name; 660 } 661 662 /* Discover virtqueues and write information to configuration. */ 663 err = virtio_find_vqs(vdev, num_vqs, vqs, callbacks, names, &desc); 664 if (err) 665 goto out; 666 667 for (i = 0; i < num_vqs; i++) { 668 spin_lock_init(&vblk->vqs[i].lock); 669 vblk->vqs[i].vq = vqs[i]; 670 } 671 vblk->num_vqs = num_vqs; 672 673 out: 674 kfree(vqs); 675 kfree(callbacks); 676 kfree(names); 677 if (err) 678 kfree(vblk->vqs); 679 return err; 680 } 681 682 /* 683 * Legacy naming scheme used for virtio devices. We are stuck with it for 684 * virtio blk but don't ever use it for any new driver. 685 */ 686 static int virtblk_name_format(char *prefix, int index, char *buf, int buflen) 687 { 688 const int base = 'z' - 'a' + 1; 689 char *begin = buf + strlen(prefix); 690 char *end = buf + buflen; 691 char *p; 692 int unit; 693 694 p = end - 1; 695 *p = '\0'; 696 unit = base; 697 do { 698 if (p == begin) 699 return -EINVAL; 700 *--p = 'a' + (index % unit); 701 index = (index / unit) - 1; 702 } while (index >= 0); 703 704 memmove(begin, p, end - p); 705 memcpy(buf, prefix, strlen(prefix)); 706 707 return 0; 708 } 709 710 static int virtblk_get_cache_mode(struct virtio_device *vdev) 711 { 712 u8 writeback; 713 int err; 714 715 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE, 716 struct virtio_blk_config, wce, 717 &writeback); 718 719 /* 720 * If WCE is not configurable and flush is not available, 721 * assume no writeback cache is in use. 722 */ 723 if (err) 724 writeback = virtio_has_feature(vdev, VIRTIO_BLK_F_FLUSH); 725 726 return writeback; 727 } 728 729 static void virtblk_update_cache_mode(struct virtio_device *vdev) 730 { 731 u8 writeback = virtblk_get_cache_mode(vdev); 732 struct virtio_blk *vblk = vdev->priv; 733 734 blk_queue_write_cache(vblk->disk->queue, writeback, false); 735 } 736 737 static const char *const virtblk_cache_types[] = { 738 "write through", "write back" 739 }; 740 741 static ssize_t 742 cache_type_store(struct device *dev, struct device_attribute *attr, 743 const char *buf, size_t count) 744 { 745 struct gendisk *disk = dev_to_disk(dev); 746 struct virtio_blk *vblk = disk->private_data; 747 struct virtio_device *vdev = vblk->vdev; 748 int i; 749 750 BUG_ON(!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_CONFIG_WCE)); 751 i = sysfs_match_string(virtblk_cache_types, buf); 752 if (i < 0) 753 return i; 754 755 virtio_cwrite8(vdev, offsetof(struct virtio_blk_config, wce), i); 756 virtblk_update_cache_mode(vdev); 757 return count; 758 } 759 760 static ssize_t 761 cache_type_show(struct device *dev, struct device_attribute *attr, char *buf) 762 { 763 struct gendisk *disk = dev_to_disk(dev); 764 struct virtio_blk *vblk = disk->private_data; 765 u8 writeback = virtblk_get_cache_mode(vblk->vdev); 766 767 BUG_ON(writeback >= ARRAY_SIZE(virtblk_cache_types)); 768 return sysfs_emit(buf, "%s\n", virtblk_cache_types[writeback]); 769 } 770 771 static DEVICE_ATTR_RW(cache_type); 772 773 static struct attribute *virtblk_attrs[] = { 774 &dev_attr_serial.attr, 775 &dev_attr_cache_type.attr, 776 NULL, 777 }; 778 779 static umode_t virtblk_attrs_are_visible(struct kobject *kobj, 780 struct attribute *a, int n) 781 { 782 struct device *dev = kobj_to_dev(kobj); 783 struct gendisk *disk = dev_to_disk(dev); 784 struct virtio_blk *vblk = disk->private_data; 785 struct virtio_device *vdev = vblk->vdev; 786 787 if (a == &dev_attr_cache_type.attr && 788 !virtio_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE)) 789 return S_IRUGO; 790 791 return a->mode; 792 } 793 794 static const struct attribute_group virtblk_attr_group = { 795 .attrs = virtblk_attrs, 796 .is_visible = virtblk_attrs_are_visible, 797 }; 798 799 static const struct attribute_group *virtblk_attr_groups[] = { 800 &virtblk_attr_group, 801 NULL, 802 }; 803 804 static int virtblk_map_queues(struct blk_mq_tag_set *set) 805 { 806 struct virtio_blk *vblk = set->driver_data; 807 int i, qoff; 808 809 for (i = 0, qoff = 0; i < set->nr_maps; i++) { 810 struct blk_mq_queue_map *map = &set->map[i]; 811 812 map->nr_queues = vblk->io_queues[i]; 813 map->queue_offset = qoff; 814 qoff += map->nr_queues; 815 816 if (map->nr_queues == 0) 817 continue; 818 819 /* 820 * Regular queues have interrupts and hence CPU affinity is 821 * defined by the core virtio code, but polling queues have 822 * no interrupts so we let the block layer assign CPU affinity. 823 */ 824 if (i == HCTX_TYPE_POLL) 825 blk_mq_map_queues(&set->map[i]); 826 else 827 blk_mq_virtio_map_queues(&set->map[i], vblk->vdev, 0); 828 } 829 830 return 0; 831 } 832 833 static void virtblk_complete_batch(struct io_comp_batch *iob) 834 { 835 struct request *req; 836 837 rq_list_for_each(&iob->req_list, req) { 838 virtblk_unmap_data(req, blk_mq_rq_to_pdu(req)); 839 virtblk_cleanup_cmd(req); 840 } 841 blk_mq_end_request_batch(iob); 842 } 843 844 static int virtblk_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) 845 { 846 struct virtio_blk *vblk = hctx->queue->queuedata; 847 struct virtio_blk_vq *vq = get_virtio_blk_vq(hctx); 848 struct virtblk_req *vbr; 849 unsigned long flags; 850 unsigned int len; 851 int found = 0; 852 853 spin_lock_irqsave(&vq->lock, flags); 854 855 while ((vbr = virtqueue_get_buf(vq->vq, &len)) != NULL) { 856 struct request *req = blk_mq_rq_from_pdu(vbr); 857 858 found++; 859 if (!blk_mq_add_to_batch(req, iob, vbr->status, 860 virtblk_complete_batch)) 861 blk_mq_complete_request(req); 862 } 863 864 if (found) 865 blk_mq_start_stopped_hw_queues(vblk->disk->queue, true); 866 867 spin_unlock_irqrestore(&vq->lock, flags); 868 869 return found; 870 } 871 872 static const struct blk_mq_ops virtio_mq_ops = { 873 .queue_rq = virtio_queue_rq, 874 .queue_rqs = virtio_queue_rqs, 875 .commit_rqs = virtio_commit_rqs, 876 .complete = virtblk_request_done, 877 .map_queues = virtblk_map_queues, 878 .poll = virtblk_poll, 879 }; 880 881 static unsigned int virtblk_queue_depth; 882 module_param_named(queue_depth, virtblk_queue_depth, uint, 0444); 883 884 static int virtblk_probe(struct virtio_device *vdev) 885 { 886 struct virtio_blk *vblk; 887 struct request_queue *q; 888 int err, index; 889 890 u32 v, blk_size, max_size, sg_elems, opt_io_size; 891 u16 min_io_size; 892 u8 physical_block_exp, alignment_offset; 893 unsigned int queue_depth; 894 895 if (!vdev->config->get) { 896 dev_err(&vdev->dev, "%s failure: config access disabled\n", 897 __func__); 898 return -EINVAL; 899 } 900 901 err = ida_simple_get(&vd_index_ida, 0, minor_to_index(1 << MINORBITS), 902 GFP_KERNEL); 903 if (err < 0) 904 goto out; 905 index = err; 906 907 /* We need to know how many segments before we allocate. */ 908 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SEG_MAX, 909 struct virtio_blk_config, seg_max, 910 &sg_elems); 911 912 /* We need at least one SG element, whatever they say. */ 913 if (err || !sg_elems) 914 sg_elems = 1; 915 916 /* Prevent integer overflows and honor max vq size */ 917 sg_elems = min_t(u32, sg_elems, VIRTIO_BLK_MAX_SG_ELEMS - 2); 918 919 vdev->priv = vblk = kmalloc(sizeof(*vblk), GFP_KERNEL); 920 if (!vblk) { 921 err = -ENOMEM; 922 goto out_free_index; 923 } 924 925 mutex_init(&vblk->vdev_mutex); 926 927 vblk->vdev = vdev; 928 929 INIT_WORK(&vblk->config_work, virtblk_config_changed_work); 930 931 err = init_vq(vblk); 932 if (err) 933 goto out_free_vblk; 934 935 /* Default queue sizing is to fill the ring. */ 936 if (!virtblk_queue_depth) { 937 queue_depth = vblk->vqs[0].vq->num_free; 938 /* ... but without indirect descs, we use 2 descs per req */ 939 if (!virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) 940 queue_depth /= 2; 941 } else { 942 queue_depth = virtblk_queue_depth; 943 } 944 945 memset(&vblk->tag_set, 0, sizeof(vblk->tag_set)); 946 vblk->tag_set.ops = &virtio_mq_ops; 947 vblk->tag_set.queue_depth = queue_depth; 948 vblk->tag_set.numa_node = NUMA_NO_NODE; 949 vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; 950 vblk->tag_set.cmd_size = 951 sizeof(struct virtblk_req) + 952 sizeof(struct scatterlist) * VIRTIO_BLK_INLINE_SG_CNT; 953 vblk->tag_set.driver_data = vblk; 954 vblk->tag_set.nr_hw_queues = vblk->num_vqs; 955 vblk->tag_set.nr_maps = 1; 956 if (vblk->io_queues[HCTX_TYPE_POLL]) 957 vblk->tag_set.nr_maps = 3; 958 959 err = blk_mq_alloc_tag_set(&vblk->tag_set); 960 if (err) 961 goto out_free_vq; 962 963 vblk->disk = blk_mq_alloc_disk(&vblk->tag_set, vblk); 964 if (IS_ERR(vblk->disk)) { 965 err = PTR_ERR(vblk->disk); 966 goto out_free_tags; 967 } 968 q = vblk->disk->queue; 969 970 virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN); 971 972 vblk->disk->major = major; 973 vblk->disk->first_minor = index_to_minor(index); 974 vblk->disk->minors = 1 << PART_BITS; 975 vblk->disk->private_data = vblk; 976 vblk->disk->fops = &virtblk_fops; 977 vblk->index = index; 978 979 /* configure queue flush support */ 980 virtblk_update_cache_mode(vdev); 981 982 /* If disk is read-only in the host, the guest should obey */ 983 if (virtio_has_feature(vdev, VIRTIO_BLK_F_RO)) 984 set_disk_ro(vblk->disk, 1); 985 986 /* We can handle whatever the host told us to handle. */ 987 blk_queue_max_segments(q, sg_elems); 988 989 /* No real sector limit. */ 990 blk_queue_max_hw_sectors(q, -1U); 991 992 max_size = virtio_max_dma_size(vdev); 993 994 /* Host can optionally specify maximum segment size and number of 995 * segments. */ 996 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SIZE_MAX, 997 struct virtio_blk_config, size_max, &v); 998 if (!err) 999 max_size = min(max_size, v); 1000 1001 blk_queue_max_segment_size(q, max_size); 1002 1003 /* Host can optionally specify the block size of the device */ 1004 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_BLK_SIZE, 1005 struct virtio_blk_config, blk_size, 1006 &blk_size); 1007 if (!err) { 1008 err = blk_validate_block_size(blk_size); 1009 if (err) { 1010 dev_err(&vdev->dev, 1011 "virtio_blk: invalid block size: 0x%x\n", 1012 blk_size); 1013 goto out_cleanup_disk; 1014 } 1015 1016 blk_queue_logical_block_size(q, blk_size); 1017 } else 1018 blk_size = queue_logical_block_size(q); 1019 1020 /* Use topology information if available */ 1021 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 1022 struct virtio_blk_config, physical_block_exp, 1023 &physical_block_exp); 1024 if (!err && physical_block_exp) 1025 blk_queue_physical_block_size(q, 1026 blk_size * (1 << physical_block_exp)); 1027 1028 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 1029 struct virtio_blk_config, alignment_offset, 1030 &alignment_offset); 1031 if (!err && alignment_offset) 1032 blk_queue_alignment_offset(q, blk_size * alignment_offset); 1033 1034 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 1035 struct virtio_blk_config, min_io_size, 1036 &min_io_size); 1037 if (!err && min_io_size) 1038 blk_queue_io_min(q, blk_size * min_io_size); 1039 1040 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 1041 struct virtio_blk_config, opt_io_size, 1042 &opt_io_size); 1043 if (!err && opt_io_size) 1044 blk_queue_io_opt(q, blk_size * opt_io_size); 1045 1046 if (virtio_has_feature(vdev, VIRTIO_BLK_F_DISCARD)) { 1047 virtio_cread(vdev, struct virtio_blk_config, 1048 discard_sector_alignment, &v); 1049 if (v) 1050 q->limits.discard_granularity = v << SECTOR_SHIFT; 1051 else 1052 q->limits.discard_granularity = blk_size; 1053 1054 virtio_cread(vdev, struct virtio_blk_config, 1055 max_discard_sectors, &v); 1056 blk_queue_max_discard_sectors(q, v ? v : UINT_MAX); 1057 1058 virtio_cread(vdev, struct virtio_blk_config, max_discard_seg, 1059 &v); 1060 1061 /* 1062 * max_discard_seg == 0 is out of spec but we always 1063 * handled it. 1064 */ 1065 if (!v) 1066 v = sg_elems; 1067 blk_queue_max_discard_segments(q, 1068 min(v, MAX_DISCARD_SEGMENTS)); 1069 } 1070 1071 if (virtio_has_feature(vdev, VIRTIO_BLK_F_WRITE_ZEROES)) { 1072 virtio_cread(vdev, struct virtio_blk_config, 1073 max_write_zeroes_sectors, &v); 1074 blk_queue_max_write_zeroes_sectors(q, v ? v : UINT_MAX); 1075 } 1076 1077 virtblk_update_capacity(vblk, false); 1078 virtio_device_ready(vdev); 1079 1080 err = device_add_disk(&vdev->dev, vblk->disk, virtblk_attr_groups); 1081 if (err) 1082 goto out_cleanup_disk; 1083 1084 return 0; 1085 1086 out_cleanup_disk: 1087 put_disk(vblk->disk); 1088 out_free_tags: 1089 blk_mq_free_tag_set(&vblk->tag_set); 1090 out_free_vq: 1091 vdev->config->del_vqs(vdev); 1092 kfree(vblk->vqs); 1093 out_free_vblk: 1094 kfree(vblk); 1095 out_free_index: 1096 ida_simple_remove(&vd_index_ida, index); 1097 out: 1098 return err; 1099 } 1100 1101 static void virtblk_remove(struct virtio_device *vdev) 1102 { 1103 struct virtio_blk *vblk = vdev->priv; 1104 1105 /* Make sure no work handler is accessing the device. */ 1106 flush_work(&vblk->config_work); 1107 1108 del_gendisk(vblk->disk); 1109 blk_mq_free_tag_set(&vblk->tag_set); 1110 1111 mutex_lock(&vblk->vdev_mutex); 1112 1113 /* Stop all the virtqueues. */ 1114 virtio_reset_device(vdev); 1115 1116 /* Virtqueues are stopped, nothing can use vblk->vdev anymore. */ 1117 vblk->vdev = NULL; 1118 1119 vdev->config->del_vqs(vdev); 1120 kfree(vblk->vqs); 1121 1122 mutex_unlock(&vblk->vdev_mutex); 1123 1124 put_disk(vblk->disk); 1125 } 1126 1127 #ifdef CONFIG_PM_SLEEP 1128 static int virtblk_freeze(struct virtio_device *vdev) 1129 { 1130 struct virtio_blk *vblk = vdev->priv; 1131 1132 /* Ensure we don't receive any more interrupts */ 1133 virtio_reset_device(vdev); 1134 1135 /* Make sure no work handler is accessing the device. */ 1136 flush_work(&vblk->config_work); 1137 1138 blk_mq_quiesce_queue(vblk->disk->queue); 1139 1140 vdev->config->del_vqs(vdev); 1141 kfree(vblk->vqs); 1142 1143 return 0; 1144 } 1145 1146 static int virtblk_restore(struct virtio_device *vdev) 1147 { 1148 struct virtio_blk *vblk = vdev->priv; 1149 int ret; 1150 1151 ret = init_vq(vdev->priv); 1152 if (ret) 1153 return ret; 1154 1155 virtio_device_ready(vdev); 1156 1157 blk_mq_unquiesce_queue(vblk->disk->queue); 1158 return 0; 1159 } 1160 #endif 1161 1162 static const struct virtio_device_id id_table[] = { 1163 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID }, 1164 { 0 }, 1165 }; 1166 1167 static unsigned int features_legacy[] = { 1168 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY, 1169 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, 1170 VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, 1171 VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES, 1172 } 1173 ; 1174 static unsigned int features[] = { 1175 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY, 1176 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, 1177 VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, 1178 VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES, 1179 }; 1180 1181 static struct virtio_driver virtio_blk = { 1182 .feature_table = features, 1183 .feature_table_size = ARRAY_SIZE(features), 1184 .feature_table_legacy = features_legacy, 1185 .feature_table_size_legacy = ARRAY_SIZE(features_legacy), 1186 .driver.name = KBUILD_MODNAME, 1187 .driver.owner = THIS_MODULE, 1188 .id_table = id_table, 1189 .probe = virtblk_probe, 1190 .remove = virtblk_remove, 1191 .config_changed = virtblk_config_changed, 1192 #ifdef CONFIG_PM_SLEEP 1193 .freeze = virtblk_freeze, 1194 .restore = virtblk_restore, 1195 #endif 1196 }; 1197 1198 static int __init virtio_blk_init(void) 1199 { 1200 int error; 1201 1202 virtblk_wq = alloc_workqueue("virtio-blk", 0, 0); 1203 if (!virtblk_wq) 1204 return -ENOMEM; 1205 1206 major = register_blkdev(0, "virtblk"); 1207 if (major < 0) { 1208 error = major; 1209 goto out_destroy_workqueue; 1210 } 1211 1212 error = register_virtio_driver(&virtio_blk); 1213 if (error) 1214 goto out_unregister_blkdev; 1215 return 0; 1216 1217 out_unregister_blkdev: 1218 unregister_blkdev(major, "virtblk"); 1219 out_destroy_workqueue: 1220 destroy_workqueue(virtblk_wq); 1221 return error; 1222 } 1223 1224 static void __exit virtio_blk_fini(void) 1225 { 1226 unregister_virtio_driver(&virtio_blk); 1227 unregister_blkdev(major, "virtblk"); 1228 destroy_workqueue(virtblk_wq); 1229 } 1230 module_init(virtio_blk_init); 1231 module_exit(virtio_blk_fini); 1232 1233 MODULE_DEVICE_TABLE(virtio, id_table); 1234 MODULE_DESCRIPTION("Virtio block driver"); 1235 MODULE_LICENSE("GPL"); 1236