1 //#define DEBUG 2 #include <linux/spinlock.h> 3 #include <linux/slab.h> 4 #include <linux/blkdev.h> 5 #include <linux/hdreg.h> 6 #include <linux/module.h> 7 #include <linux/mutex.h> 8 #include <linux/interrupt.h> 9 #include <linux/virtio.h> 10 #include <linux/virtio_blk.h> 11 #include <linux/scatterlist.h> 12 #include <linux/string_helpers.h> 13 #include <scsi/scsi_cmnd.h> 14 #include <linux/idr.h> 15 #include <linux/blk-mq.h> 16 #include <linux/blk-mq-virtio.h> 17 #include <linux/numa.h> 18 19 #define PART_BITS 4 20 #define VQ_NAME_LEN 16 21 22 static int major; 23 static DEFINE_IDA(vd_index_ida); 24 25 static struct workqueue_struct *virtblk_wq; 26 27 struct virtio_blk_vq { 28 struct virtqueue *vq; 29 spinlock_t lock; 30 char name[VQ_NAME_LEN]; 31 } ____cacheline_aligned_in_smp; 32 33 struct virtio_blk { 34 struct virtio_device *vdev; 35 36 /* The disk structure for the kernel. */ 37 struct gendisk *disk; 38 39 /* Block layer tags. */ 40 struct blk_mq_tag_set tag_set; 41 42 /* Process context for config space updates */ 43 struct work_struct config_work; 44 45 /* What host tells us, plus 2 for header & tailer. */ 46 unsigned int sg_elems; 47 48 /* Ida index - used to track minor number allocations. */ 49 int index; 50 51 /* num of vqs */ 52 int num_vqs; 53 struct virtio_blk_vq *vqs; 54 }; 55 56 struct virtblk_req { 57 #ifdef CONFIG_VIRTIO_BLK_SCSI 58 struct scsi_request sreq; /* for SCSI passthrough, must be first */ 59 u8 sense[SCSI_SENSE_BUFFERSIZE]; 60 struct virtio_scsi_inhdr in_hdr; 61 #endif 62 struct virtio_blk_outhdr out_hdr; 63 u8 status; 64 struct scatterlist sg[]; 65 }; 66 67 static inline blk_status_t virtblk_result(struct virtblk_req *vbr) 68 { 69 switch (vbr->status) { 70 case VIRTIO_BLK_S_OK: 71 return BLK_STS_OK; 72 case VIRTIO_BLK_S_UNSUPP: 73 return BLK_STS_NOTSUPP; 74 default: 75 return BLK_STS_IOERR; 76 } 77 } 78 79 /* 80 * If this is a packet command we need a couple of additional headers. Behind 81 * the normal outhdr we put a segment with the scsi command block, and before 82 * the normal inhdr we put the sense data and the inhdr with additional status 83 * information. 84 */ 85 #ifdef CONFIG_VIRTIO_BLK_SCSI 86 static int virtblk_add_req_scsi(struct virtqueue *vq, struct virtblk_req *vbr, 87 struct scatterlist *data_sg, bool have_data) 88 { 89 struct scatterlist hdr, status, cmd, sense, inhdr, *sgs[6]; 90 unsigned int num_out = 0, num_in = 0; 91 92 sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr)); 93 sgs[num_out++] = &hdr; 94 sg_init_one(&cmd, vbr->sreq.cmd, vbr->sreq.cmd_len); 95 sgs[num_out++] = &cmd; 96 97 if (have_data) { 98 if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT)) 99 sgs[num_out++] = data_sg; 100 else 101 sgs[num_out + num_in++] = data_sg; 102 } 103 104 sg_init_one(&sense, vbr->sense, SCSI_SENSE_BUFFERSIZE); 105 sgs[num_out + num_in++] = &sense; 106 sg_init_one(&inhdr, &vbr->in_hdr, sizeof(vbr->in_hdr)); 107 sgs[num_out + num_in++] = &inhdr; 108 sg_init_one(&status, &vbr->status, sizeof(vbr->status)); 109 sgs[num_out + num_in++] = &status; 110 111 return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC); 112 } 113 114 static inline void virtblk_scsi_request_done(struct request *req) 115 { 116 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); 117 struct virtio_blk *vblk = req->q->queuedata; 118 struct scsi_request *sreq = &vbr->sreq; 119 120 sreq->resid_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.residual); 121 sreq->sense_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.sense_len); 122 sreq->result = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.errors); 123 } 124 125 static int virtblk_ioctl(struct block_device *bdev, fmode_t mode, 126 unsigned int cmd, unsigned long data) 127 { 128 struct gendisk *disk = bdev->bd_disk; 129 struct virtio_blk *vblk = disk->private_data; 130 131 /* 132 * Only allow the generic SCSI ioctls if the host can support it. 133 */ 134 if (!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_SCSI)) 135 return -ENOTTY; 136 137 return scsi_cmd_blk_ioctl(bdev, mode, cmd, 138 (void __user *)data); 139 } 140 #else 141 static inline int virtblk_add_req_scsi(struct virtqueue *vq, 142 struct virtblk_req *vbr, struct scatterlist *data_sg, 143 bool have_data) 144 { 145 return -EIO; 146 } 147 static inline void virtblk_scsi_request_done(struct request *req) 148 { 149 } 150 #define virtblk_ioctl NULL 151 #endif /* CONFIG_VIRTIO_BLK_SCSI */ 152 153 static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr, 154 struct scatterlist *data_sg, bool have_data) 155 { 156 struct scatterlist hdr, status, *sgs[3]; 157 unsigned int num_out = 0, num_in = 0; 158 159 sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr)); 160 sgs[num_out++] = &hdr; 161 162 if (have_data) { 163 if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT)) 164 sgs[num_out++] = data_sg; 165 else 166 sgs[num_out + num_in++] = data_sg; 167 } 168 169 sg_init_one(&status, &vbr->status, sizeof(vbr->status)); 170 sgs[num_out + num_in++] = &status; 171 172 return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC); 173 } 174 175 static inline void virtblk_request_done(struct request *req) 176 { 177 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); 178 179 switch (req_op(req)) { 180 case REQ_OP_SCSI_IN: 181 case REQ_OP_SCSI_OUT: 182 virtblk_scsi_request_done(req); 183 break; 184 } 185 186 blk_mq_end_request(req, virtblk_result(vbr)); 187 } 188 189 static void virtblk_done(struct virtqueue *vq) 190 { 191 struct virtio_blk *vblk = vq->vdev->priv; 192 bool req_done = false; 193 int qid = vq->index; 194 struct virtblk_req *vbr; 195 unsigned long flags; 196 unsigned int len; 197 198 spin_lock_irqsave(&vblk->vqs[qid].lock, flags); 199 do { 200 virtqueue_disable_cb(vq); 201 while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) { 202 struct request *req = blk_mq_rq_from_pdu(vbr); 203 204 blk_mq_complete_request(req); 205 req_done = true; 206 } 207 if (unlikely(virtqueue_is_broken(vq))) 208 break; 209 } while (!virtqueue_enable_cb(vq)); 210 211 /* In case queue is stopped waiting for more buffers. */ 212 if (req_done) 213 blk_mq_start_stopped_hw_queues(vblk->disk->queue, true); 214 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); 215 } 216 217 static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx, 218 const struct blk_mq_queue_data *bd) 219 { 220 struct virtio_blk *vblk = hctx->queue->queuedata; 221 struct request *req = bd->rq; 222 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); 223 unsigned long flags; 224 unsigned int num; 225 int qid = hctx->queue_num; 226 int err; 227 bool notify = false; 228 u32 type; 229 230 BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems); 231 232 switch (req_op(req)) { 233 case REQ_OP_READ: 234 case REQ_OP_WRITE: 235 type = 0; 236 break; 237 case REQ_OP_FLUSH: 238 type = VIRTIO_BLK_T_FLUSH; 239 break; 240 case REQ_OP_SCSI_IN: 241 case REQ_OP_SCSI_OUT: 242 type = VIRTIO_BLK_T_SCSI_CMD; 243 break; 244 case REQ_OP_DRV_IN: 245 type = VIRTIO_BLK_T_GET_ID; 246 break; 247 default: 248 WARN_ON_ONCE(1); 249 return BLK_STS_IOERR; 250 } 251 252 vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, type); 253 vbr->out_hdr.sector = type ? 254 0 : cpu_to_virtio64(vblk->vdev, blk_rq_pos(req)); 255 vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(req)); 256 257 blk_mq_start_request(req); 258 259 num = blk_rq_map_sg(hctx->queue, req, vbr->sg); 260 if (num) { 261 if (rq_data_dir(req) == WRITE) 262 vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_OUT); 263 else 264 vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_IN); 265 } 266 267 spin_lock_irqsave(&vblk->vqs[qid].lock, flags); 268 if (blk_rq_is_scsi(req)) 269 err = virtblk_add_req_scsi(vblk->vqs[qid].vq, vbr, vbr->sg, num); 270 else 271 err = virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg, num); 272 if (err) { 273 virtqueue_kick(vblk->vqs[qid].vq); 274 blk_mq_stop_hw_queue(hctx); 275 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); 276 /* Out of mem doesn't actually happen, since we fall back 277 * to direct descriptors */ 278 if (err == -ENOMEM || err == -ENOSPC) 279 return BLK_STS_DEV_RESOURCE; 280 return BLK_STS_IOERR; 281 } 282 283 if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq)) 284 notify = true; 285 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); 286 287 if (notify) 288 virtqueue_notify(vblk->vqs[qid].vq); 289 return BLK_STS_OK; 290 } 291 292 /* return id (s/n) string for *disk to *id_str 293 */ 294 static int virtblk_get_id(struct gendisk *disk, char *id_str) 295 { 296 struct virtio_blk *vblk = disk->private_data; 297 struct request_queue *q = vblk->disk->queue; 298 struct request *req; 299 int err; 300 301 req = blk_get_request(q, REQ_OP_DRV_IN, 0); 302 if (IS_ERR(req)) 303 return PTR_ERR(req); 304 305 err = blk_rq_map_kern(q, req, id_str, VIRTIO_BLK_ID_BYTES, GFP_KERNEL); 306 if (err) 307 goto out; 308 309 blk_execute_rq(vblk->disk->queue, vblk->disk, req, false); 310 err = blk_status_to_errno(virtblk_result(blk_mq_rq_to_pdu(req))); 311 out: 312 blk_put_request(req); 313 return err; 314 } 315 316 /* We provide getgeo only to please some old bootloader/partitioning tools */ 317 static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo) 318 { 319 struct virtio_blk *vblk = bd->bd_disk->private_data; 320 321 /* see if the host passed in geometry config */ 322 if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) { 323 virtio_cread(vblk->vdev, struct virtio_blk_config, 324 geometry.cylinders, &geo->cylinders); 325 virtio_cread(vblk->vdev, struct virtio_blk_config, 326 geometry.heads, &geo->heads); 327 virtio_cread(vblk->vdev, struct virtio_blk_config, 328 geometry.sectors, &geo->sectors); 329 } else { 330 /* some standard values, similar to sd */ 331 geo->heads = 1 << 6; 332 geo->sectors = 1 << 5; 333 geo->cylinders = get_capacity(bd->bd_disk) >> 11; 334 } 335 return 0; 336 } 337 338 static const struct block_device_operations virtblk_fops = { 339 .ioctl = virtblk_ioctl, 340 .owner = THIS_MODULE, 341 .getgeo = virtblk_getgeo, 342 }; 343 344 static int index_to_minor(int index) 345 { 346 return index << PART_BITS; 347 } 348 349 static int minor_to_index(int minor) 350 { 351 return minor >> PART_BITS; 352 } 353 354 static ssize_t serial_show(struct device *dev, 355 struct device_attribute *attr, char *buf) 356 { 357 struct gendisk *disk = dev_to_disk(dev); 358 int err; 359 360 /* sysfs gives us a PAGE_SIZE buffer */ 361 BUILD_BUG_ON(PAGE_SIZE < VIRTIO_BLK_ID_BYTES); 362 363 buf[VIRTIO_BLK_ID_BYTES] = '\0'; 364 err = virtblk_get_id(disk, buf); 365 if (!err) 366 return strlen(buf); 367 368 if (err == -EIO) /* Unsupported? Make it empty. */ 369 return 0; 370 371 return err; 372 } 373 374 static DEVICE_ATTR_RO(serial); 375 376 /* The queue's logical block size must be set before calling this */ 377 static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize) 378 { 379 struct virtio_device *vdev = vblk->vdev; 380 struct request_queue *q = vblk->disk->queue; 381 char cap_str_2[10], cap_str_10[10]; 382 unsigned long long nblocks; 383 u64 capacity; 384 385 /* Host must always specify the capacity. */ 386 virtio_cread(vdev, struct virtio_blk_config, capacity, &capacity); 387 388 /* If capacity is too big, truncate with warning. */ 389 if ((sector_t)capacity != capacity) { 390 dev_warn(&vdev->dev, "Capacity %llu too large: truncating\n", 391 (unsigned long long)capacity); 392 capacity = (sector_t)-1; 393 } 394 395 nblocks = DIV_ROUND_UP_ULL(capacity, queue_logical_block_size(q) >> 9); 396 397 string_get_size(nblocks, queue_logical_block_size(q), 398 STRING_UNITS_2, cap_str_2, sizeof(cap_str_2)); 399 string_get_size(nblocks, queue_logical_block_size(q), 400 STRING_UNITS_10, cap_str_10, sizeof(cap_str_10)); 401 402 dev_notice(&vdev->dev, 403 "[%s] %s%llu %d-byte logical blocks (%s/%s)\n", 404 vblk->disk->disk_name, 405 resize ? "new size: " : "", 406 nblocks, 407 queue_logical_block_size(q), 408 cap_str_10, 409 cap_str_2); 410 411 set_capacity(vblk->disk, capacity); 412 } 413 414 static void virtblk_config_changed_work(struct work_struct *work) 415 { 416 struct virtio_blk *vblk = 417 container_of(work, struct virtio_blk, config_work); 418 char *envp[] = { "RESIZE=1", NULL }; 419 420 virtblk_update_capacity(vblk, true); 421 revalidate_disk(vblk->disk); 422 kobject_uevent_env(&disk_to_dev(vblk->disk)->kobj, KOBJ_CHANGE, envp); 423 } 424 425 static void virtblk_config_changed(struct virtio_device *vdev) 426 { 427 struct virtio_blk *vblk = vdev->priv; 428 429 queue_work(virtblk_wq, &vblk->config_work); 430 } 431 432 static int init_vq(struct virtio_blk *vblk) 433 { 434 int err; 435 int i; 436 vq_callback_t **callbacks; 437 const char **names; 438 struct virtqueue **vqs; 439 unsigned short num_vqs; 440 struct virtio_device *vdev = vblk->vdev; 441 struct irq_affinity desc = { 0, }; 442 443 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_MQ, 444 struct virtio_blk_config, num_queues, 445 &num_vqs); 446 if (err) 447 num_vqs = 1; 448 449 vblk->vqs = kmalloc_array(num_vqs, sizeof(*vblk->vqs), GFP_KERNEL); 450 if (!vblk->vqs) 451 return -ENOMEM; 452 453 names = kmalloc_array(num_vqs, sizeof(*names), GFP_KERNEL); 454 callbacks = kmalloc_array(num_vqs, sizeof(*callbacks), GFP_KERNEL); 455 vqs = kmalloc_array(num_vqs, sizeof(*vqs), GFP_KERNEL); 456 if (!names || !callbacks || !vqs) { 457 err = -ENOMEM; 458 goto out; 459 } 460 461 for (i = 0; i < num_vqs; i++) { 462 callbacks[i] = virtblk_done; 463 snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%d", i); 464 names[i] = vblk->vqs[i].name; 465 } 466 467 /* Discover virtqueues and write information to configuration. */ 468 err = virtio_find_vqs(vdev, num_vqs, vqs, callbacks, names, &desc); 469 if (err) 470 goto out; 471 472 for (i = 0; i < num_vqs; i++) { 473 spin_lock_init(&vblk->vqs[i].lock); 474 vblk->vqs[i].vq = vqs[i]; 475 } 476 vblk->num_vqs = num_vqs; 477 478 out: 479 kfree(vqs); 480 kfree(callbacks); 481 kfree(names); 482 if (err) 483 kfree(vblk->vqs); 484 return err; 485 } 486 487 /* 488 * Legacy naming scheme used for virtio devices. We are stuck with it for 489 * virtio blk but don't ever use it for any new driver. 490 */ 491 static int virtblk_name_format(char *prefix, int index, char *buf, int buflen) 492 { 493 const int base = 'z' - 'a' + 1; 494 char *begin = buf + strlen(prefix); 495 char *end = buf + buflen; 496 char *p; 497 int unit; 498 499 p = end - 1; 500 *p = '\0'; 501 unit = base; 502 do { 503 if (p == begin) 504 return -EINVAL; 505 *--p = 'a' + (index % unit); 506 index = (index / unit) - 1; 507 } while (index >= 0); 508 509 memmove(begin, p, end - p); 510 memcpy(buf, prefix, strlen(prefix)); 511 512 return 0; 513 } 514 515 static int virtblk_get_cache_mode(struct virtio_device *vdev) 516 { 517 u8 writeback; 518 int err; 519 520 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE, 521 struct virtio_blk_config, wce, 522 &writeback); 523 524 /* 525 * If WCE is not configurable and flush is not available, 526 * assume no writeback cache is in use. 527 */ 528 if (err) 529 writeback = virtio_has_feature(vdev, VIRTIO_BLK_F_FLUSH); 530 531 return writeback; 532 } 533 534 static void virtblk_update_cache_mode(struct virtio_device *vdev) 535 { 536 u8 writeback = virtblk_get_cache_mode(vdev); 537 struct virtio_blk *vblk = vdev->priv; 538 539 blk_queue_write_cache(vblk->disk->queue, writeback, false); 540 revalidate_disk(vblk->disk); 541 } 542 543 static const char *const virtblk_cache_types[] = { 544 "write through", "write back" 545 }; 546 547 static ssize_t 548 cache_type_store(struct device *dev, struct device_attribute *attr, 549 const char *buf, size_t count) 550 { 551 struct gendisk *disk = dev_to_disk(dev); 552 struct virtio_blk *vblk = disk->private_data; 553 struct virtio_device *vdev = vblk->vdev; 554 int i; 555 556 BUG_ON(!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_CONFIG_WCE)); 557 i = sysfs_match_string(virtblk_cache_types, buf); 558 if (i < 0) 559 return i; 560 561 virtio_cwrite8(vdev, offsetof(struct virtio_blk_config, wce), i); 562 virtblk_update_cache_mode(vdev); 563 return count; 564 } 565 566 static ssize_t 567 cache_type_show(struct device *dev, struct device_attribute *attr, char *buf) 568 { 569 struct gendisk *disk = dev_to_disk(dev); 570 struct virtio_blk *vblk = disk->private_data; 571 u8 writeback = virtblk_get_cache_mode(vblk->vdev); 572 573 BUG_ON(writeback >= ARRAY_SIZE(virtblk_cache_types)); 574 return snprintf(buf, 40, "%s\n", virtblk_cache_types[writeback]); 575 } 576 577 static DEVICE_ATTR_RW(cache_type); 578 579 static struct attribute *virtblk_attrs[] = { 580 &dev_attr_serial.attr, 581 &dev_attr_cache_type.attr, 582 NULL, 583 }; 584 585 static umode_t virtblk_attrs_are_visible(struct kobject *kobj, 586 struct attribute *a, int n) 587 { 588 struct device *dev = container_of(kobj, struct device, kobj); 589 struct gendisk *disk = dev_to_disk(dev); 590 struct virtio_blk *vblk = disk->private_data; 591 struct virtio_device *vdev = vblk->vdev; 592 593 if (a == &dev_attr_cache_type.attr && 594 !virtio_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE)) 595 return S_IRUGO; 596 597 return a->mode; 598 } 599 600 static const struct attribute_group virtblk_attr_group = { 601 .attrs = virtblk_attrs, 602 .is_visible = virtblk_attrs_are_visible, 603 }; 604 605 static const struct attribute_group *virtblk_attr_groups[] = { 606 &virtblk_attr_group, 607 NULL, 608 }; 609 610 static int virtblk_init_request(struct blk_mq_tag_set *set, struct request *rq, 611 unsigned int hctx_idx, unsigned int numa_node) 612 { 613 struct virtio_blk *vblk = set->driver_data; 614 struct virtblk_req *vbr = blk_mq_rq_to_pdu(rq); 615 616 #ifdef CONFIG_VIRTIO_BLK_SCSI 617 vbr->sreq.sense = vbr->sense; 618 #endif 619 sg_init_table(vbr->sg, vblk->sg_elems); 620 return 0; 621 } 622 623 static int virtblk_map_queues(struct blk_mq_tag_set *set) 624 { 625 struct virtio_blk *vblk = set->driver_data; 626 627 return blk_mq_virtio_map_queues(set, vblk->vdev, 0); 628 } 629 630 #ifdef CONFIG_VIRTIO_BLK_SCSI 631 static void virtblk_initialize_rq(struct request *req) 632 { 633 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); 634 635 scsi_req_init(&vbr->sreq); 636 } 637 #endif 638 639 static const struct blk_mq_ops virtio_mq_ops = { 640 .queue_rq = virtio_queue_rq, 641 .complete = virtblk_request_done, 642 .init_request = virtblk_init_request, 643 #ifdef CONFIG_VIRTIO_BLK_SCSI 644 .initialize_rq_fn = virtblk_initialize_rq, 645 #endif 646 .map_queues = virtblk_map_queues, 647 }; 648 649 static unsigned int virtblk_queue_depth; 650 module_param_named(queue_depth, virtblk_queue_depth, uint, 0444); 651 652 static int virtblk_probe(struct virtio_device *vdev) 653 { 654 struct virtio_blk *vblk; 655 struct request_queue *q; 656 int err, index; 657 658 u32 v, blk_size, sg_elems, opt_io_size; 659 u16 min_io_size; 660 u8 physical_block_exp, alignment_offset; 661 662 if (!vdev->config->get) { 663 dev_err(&vdev->dev, "%s failure: config access disabled\n", 664 __func__); 665 return -EINVAL; 666 } 667 668 err = ida_simple_get(&vd_index_ida, 0, minor_to_index(1 << MINORBITS), 669 GFP_KERNEL); 670 if (err < 0) 671 goto out; 672 index = err; 673 674 /* We need to know how many segments before we allocate. */ 675 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SEG_MAX, 676 struct virtio_blk_config, seg_max, 677 &sg_elems); 678 679 /* We need at least one SG element, whatever they say. */ 680 if (err || !sg_elems) 681 sg_elems = 1; 682 683 /* We need an extra sg elements at head and tail. */ 684 sg_elems += 2; 685 vdev->priv = vblk = kmalloc(sizeof(*vblk), GFP_KERNEL); 686 if (!vblk) { 687 err = -ENOMEM; 688 goto out_free_index; 689 } 690 691 vblk->vdev = vdev; 692 vblk->sg_elems = sg_elems; 693 694 INIT_WORK(&vblk->config_work, virtblk_config_changed_work); 695 696 err = init_vq(vblk); 697 if (err) 698 goto out_free_vblk; 699 700 /* FIXME: How many partitions? How long is a piece of string? */ 701 vblk->disk = alloc_disk(1 << PART_BITS); 702 if (!vblk->disk) { 703 err = -ENOMEM; 704 goto out_free_vq; 705 } 706 707 /* Default queue sizing is to fill the ring. */ 708 if (!virtblk_queue_depth) { 709 virtblk_queue_depth = vblk->vqs[0].vq->num_free; 710 /* ... but without indirect descs, we use 2 descs per req */ 711 if (!virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) 712 virtblk_queue_depth /= 2; 713 } 714 715 memset(&vblk->tag_set, 0, sizeof(vblk->tag_set)); 716 vblk->tag_set.ops = &virtio_mq_ops; 717 vblk->tag_set.queue_depth = virtblk_queue_depth; 718 vblk->tag_set.numa_node = NUMA_NO_NODE; 719 vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; 720 vblk->tag_set.cmd_size = 721 sizeof(struct virtblk_req) + 722 sizeof(struct scatterlist) * sg_elems; 723 vblk->tag_set.driver_data = vblk; 724 vblk->tag_set.nr_hw_queues = vblk->num_vqs; 725 726 err = blk_mq_alloc_tag_set(&vblk->tag_set); 727 if (err) 728 goto out_put_disk; 729 730 q = blk_mq_init_queue(&vblk->tag_set); 731 if (IS_ERR(q)) { 732 err = -ENOMEM; 733 goto out_free_tags; 734 } 735 vblk->disk->queue = q; 736 737 q->queuedata = vblk; 738 739 virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN); 740 741 vblk->disk->major = major; 742 vblk->disk->first_minor = index_to_minor(index); 743 vblk->disk->private_data = vblk; 744 vblk->disk->fops = &virtblk_fops; 745 vblk->disk->flags |= GENHD_FL_EXT_DEVT; 746 vblk->index = index; 747 748 /* configure queue flush support */ 749 virtblk_update_cache_mode(vdev); 750 751 /* If disk is read-only in the host, the guest should obey */ 752 if (virtio_has_feature(vdev, VIRTIO_BLK_F_RO)) 753 set_disk_ro(vblk->disk, 1); 754 755 /* We can handle whatever the host told us to handle. */ 756 blk_queue_max_segments(q, vblk->sg_elems-2); 757 758 /* No real sector limit. */ 759 blk_queue_max_hw_sectors(q, -1U); 760 761 /* Host can optionally specify maximum segment size and number of 762 * segments. */ 763 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SIZE_MAX, 764 struct virtio_blk_config, size_max, &v); 765 if (!err) 766 blk_queue_max_segment_size(q, v); 767 else 768 blk_queue_max_segment_size(q, -1U); 769 770 /* Host can optionally specify the block size of the device */ 771 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_BLK_SIZE, 772 struct virtio_blk_config, blk_size, 773 &blk_size); 774 if (!err) 775 blk_queue_logical_block_size(q, blk_size); 776 else 777 blk_size = queue_logical_block_size(q); 778 779 /* Use topology information if available */ 780 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 781 struct virtio_blk_config, physical_block_exp, 782 &physical_block_exp); 783 if (!err && physical_block_exp) 784 blk_queue_physical_block_size(q, 785 blk_size * (1 << physical_block_exp)); 786 787 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 788 struct virtio_blk_config, alignment_offset, 789 &alignment_offset); 790 if (!err && alignment_offset) 791 blk_queue_alignment_offset(q, blk_size * alignment_offset); 792 793 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 794 struct virtio_blk_config, min_io_size, 795 &min_io_size); 796 if (!err && min_io_size) 797 blk_queue_io_min(q, blk_size * min_io_size); 798 799 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 800 struct virtio_blk_config, opt_io_size, 801 &opt_io_size); 802 if (!err && opt_io_size) 803 blk_queue_io_opt(q, blk_size * opt_io_size); 804 805 virtblk_update_capacity(vblk, false); 806 virtio_device_ready(vdev); 807 808 device_add_disk(&vdev->dev, vblk->disk, virtblk_attr_groups); 809 return 0; 810 811 out_free_tags: 812 blk_mq_free_tag_set(&vblk->tag_set); 813 out_put_disk: 814 put_disk(vblk->disk); 815 out_free_vq: 816 vdev->config->del_vqs(vdev); 817 out_free_vblk: 818 kfree(vblk); 819 out_free_index: 820 ida_simple_remove(&vd_index_ida, index); 821 out: 822 return err; 823 } 824 825 static void virtblk_remove(struct virtio_device *vdev) 826 { 827 struct virtio_blk *vblk = vdev->priv; 828 int index = vblk->index; 829 int refc; 830 831 /* Make sure no work handler is accessing the device. */ 832 flush_work(&vblk->config_work); 833 834 del_gendisk(vblk->disk); 835 blk_cleanup_queue(vblk->disk->queue); 836 837 blk_mq_free_tag_set(&vblk->tag_set); 838 839 /* Stop all the virtqueues. */ 840 vdev->config->reset(vdev); 841 842 refc = kref_read(&disk_to_dev(vblk->disk)->kobj.kref); 843 put_disk(vblk->disk); 844 vdev->config->del_vqs(vdev); 845 kfree(vblk->vqs); 846 kfree(vblk); 847 848 /* Only free device id if we don't have any users */ 849 if (refc == 1) 850 ida_simple_remove(&vd_index_ida, index); 851 } 852 853 #ifdef CONFIG_PM_SLEEP 854 static int virtblk_freeze(struct virtio_device *vdev) 855 { 856 struct virtio_blk *vblk = vdev->priv; 857 858 /* Ensure we don't receive any more interrupts */ 859 vdev->config->reset(vdev); 860 861 /* Make sure no work handler is accessing the device. */ 862 flush_work(&vblk->config_work); 863 864 blk_mq_quiesce_queue(vblk->disk->queue); 865 866 vdev->config->del_vqs(vdev); 867 return 0; 868 } 869 870 static int virtblk_restore(struct virtio_device *vdev) 871 { 872 struct virtio_blk *vblk = vdev->priv; 873 int ret; 874 875 ret = init_vq(vdev->priv); 876 if (ret) 877 return ret; 878 879 virtio_device_ready(vdev); 880 881 blk_mq_unquiesce_queue(vblk->disk->queue); 882 return 0; 883 } 884 #endif 885 886 static const struct virtio_device_id id_table[] = { 887 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID }, 888 { 0 }, 889 }; 890 891 static unsigned int features_legacy[] = { 892 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY, 893 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, 894 #ifdef CONFIG_VIRTIO_BLK_SCSI 895 VIRTIO_BLK_F_SCSI, 896 #endif 897 VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, 898 VIRTIO_BLK_F_MQ, 899 } 900 ; 901 static unsigned int features[] = { 902 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY, 903 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, 904 VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, 905 VIRTIO_BLK_F_MQ, 906 }; 907 908 static struct virtio_driver virtio_blk = { 909 .feature_table = features, 910 .feature_table_size = ARRAY_SIZE(features), 911 .feature_table_legacy = features_legacy, 912 .feature_table_size_legacy = ARRAY_SIZE(features_legacy), 913 .driver.name = KBUILD_MODNAME, 914 .driver.owner = THIS_MODULE, 915 .id_table = id_table, 916 .probe = virtblk_probe, 917 .remove = virtblk_remove, 918 .config_changed = virtblk_config_changed, 919 #ifdef CONFIG_PM_SLEEP 920 .freeze = virtblk_freeze, 921 .restore = virtblk_restore, 922 #endif 923 }; 924 925 static int __init init(void) 926 { 927 int error; 928 929 virtblk_wq = alloc_workqueue("virtio-blk", 0, 0); 930 if (!virtblk_wq) 931 return -ENOMEM; 932 933 major = register_blkdev(0, "virtblk"); 934 if (major < 0) { 935 error = major; 936 goto out_destroy_workqueue; 937 } 938 939 error = register_virtio_driver(&virtio_blk); 940 if (error) 941 goto out_unregister_blkdev; 942 return 0; 943 944 out_unregister_blkdev: 945 unregister_blkdev(major, "virtblk"); 946 out_destroy_workqueue: 947 destroy_workqueue(virtblk_wq); 948 return error; 949 } 950 951 static void __exit fini(void) 952 { 953 unregister_virtio_driver(&virtio_blk); 954 unregister_blkdev(major, "virtblk"); 955 destroy_workqueue(virtblk_wq); 956 } 957 module_init(init); 958 module_exit(fini); 959 960 MODULE_DEVICE_TABLE(virtio, id_table); 961 MODULE_DESCRIPTION("Virtio block driver"); 962 MODULE_LICENSE("GPL"); 963