1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VDPA simulator for block device. 4 * 5 * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. 6 * Copyright (c) 2021, Red Hat Inc. All rights reserved. 7 * 8 */ 9 10 #include <linux/init.h> 11 #include <linux/module.h> 12 #include <linux/device.h> 13 #include <linux/kernel.h> 14 #include <linux/blkdev.h> 15 #include <linux/vringh.h> 16 #include <linux/vdpa.h> 17 #include <uapi/linux/virtio_blk.h> 18 19 #include "vdpa_sim.h" 20 21 #define DRV_VERSION "0.1" 22 #define DRV_AUTHOR "Max Gurtovoy <mgurtovoy@nvidia.com>" 23 #define DRV_DESC "vDPA Device Simulator for block device" 24 #define DRV_LICENSE "GPL v2" 25 26 #define VDPASIM_BLK_FEATURES (VDPASIM_FEATURES | \ 27 (1ULL << VIRTIO_BLK_F_FLUSH) | \ 28 (1ULL << VIRTIO_BLK_F_SIZE_MAX) | \ 29 (1ULL << VIRTIO_BLK_F_SEG_MAX) | \ 30 (1ULL << VIRTIO_BLK_F_BLK_SIZE) | \ 31 (1ULL << VIRTIO_BLK_F_TOPOLOGY) | \ 32 (1ULL << VIRTIO_BLK_F_MQ) | \ 33 (1ULL << VIRTIO_BLK_F_DISCARD) | \ 34 (1ULL << VIRTIO_BLK_F_WRITE_ZEROES)) 35 36 #define VDPASIM_BLK_CAPACITY 0x40000 37 #define VDPASIM_BLK_SIZE_MAX 0x1000 38 #define VDPASIM_BLK_SEG_MAX 32 39 #define VDPASIM_BLK_DWZ_MAX_SECTORS UINT_MAX 40 41 /* 1 virtqueue, 1 address space, 1 virtqueue group */ 42 #define VDPASIM_BLK_VQ_NUM 1 43 #define VDPASIM_BLK_AS_NUM 1 44 #define VDPASIM_BLK_GROUP_NUM 1 45 46 struct vdpasim_blk { 47 struct vdpasim vdpasim; 48 void *buffer; 49 bool shared_backend; 50 }; 51 52 static struct vdpasim_blk *sim_to_blk(struct vdpasim *vdpasim) 53 { 54 return container_of(vdpasim, struct vdpasim_blk, vdpasim); 55 } 56 57 static char vdpasim_blk_id[VIRTIO_BLK_ID_BYTES] = "vdpa_blk_sim"; 58 59 static bool shared_backend; 60 module_param(shared_backend, bool, 0444); 61 MODULE_PARM_DESC(shared_backend, "Enable the shared backend between virtio-blk devices"); 62 63 static void *shared_buffer; 64 /* mutex to synchronize shared_buffer access */ 65 static DEFINE_MUTEX(shared_buffer_mutex); 66 67 static void vdpasim_blk_buffer_lock(struct vdpasim_blk *blk) 68 { 69 if (blk->shared_backend) 70 mutex_lock(&shared_buffer_mutex); 71 } 72 73 static void vdpasim_blk_buffer_unlock(struct vdpasim_blk *blk) 74 { 75 if (blk->shared_backend) 76 mutex_unlock(&shared_buffer_mutex); 77 } 78 79 static bool vdpasim_blk_check_range(struct vdpasim *vdpasim, u64 start_sector, 80 u64 num_sectors, u64 max_sectors) 81 { 82 if (start_sector > VDPASIM_BLK_CAPACITY) { 83 dev_dbg(&vdpasim->vdpa.dev, 84 "starting sector exceeds the capacity - start: 0x%llx capacity: 0x%x\n", 85 start_sector, VDPASIM_BLK_CAPACITY); 86 } 87 88 if (num_sectors > max_sectors) { 89 dev_dbg(&vdpasim->vdpa.dev, 90 "number of sectors exceeds the max allowed in a request - num: 0x%llx max: 0x%llx\n", 91 num_sectors, max_sectors); 92 return false; 93 } 94 95 if (num_sectors > VDPASIM_BLK_CAPACITY - start_sector) { 96 dev_dbg(&vdpasim->vdpa.dev, 97 "request exceeds the capacity - start: 0x%llx num: 0x%llx capacity: 0x%x\n", 98 start_sector, num_sectors, VDPASIM_BLK_CAPACITY); 99 return false; 100 } 101 102 return true; 103 } 104 105 /* Returns 'true' if the request is handled (with or without an I/O error) 106 * and the status is correctly written in the last byte of the 'in iov', 107 * 'false' otherwise. 108 */ 109 static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, 110 struct vdpasim_virtqueue *vq) 111 { 112 struct vdpasim_blk *blk = sim_to_blk(vdpasim); 113 size_t pushed = 0, to_pull, to_push; 114 struct virtio_blk_outhdr hdr; 115 bool handled = false; 116 ssize_t bytes; 117 loff_t offset; 118 u64 sector; 119 u8 status; 120 u32 type; 121 int ret; 122 123 ret = vringh_getdesc_iotlb(&vq->vring, &vq->out_iov, &vq->in_iov, 124 &vq->head, GFP_ATOMIC); 125 if (ret != 1) 126 return false; 127 128 if (vq->out_iov.used < 1 || vq->in_iov.used < 1) { 129 dev_dbg(&vdpasim->vdpa.dev, "missing headers - out_iov: %u in_iov %u\n", 130 vq->out_iov.used, vq->in_iov.used); 131 goto err; 132 } 133 134 if (vq->in_iov.iov[vq->in_iov.used - 1].iov_len < 1) { 135 dev_dbg(&vdpasim->vdpa.dev, "request in header too short\n"); 136 goto err; 137 } 138 139 /* The last byte is the status and we checked if the last iov has 140 * enough room for it. 141 */ 142 to_push = vringh_kiov_length(&vq->in_iov) - 1; 143 144 to_pull = vringh_kiov_length(&vq->out_iov); 145 146 bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov, &hdr, 147 sizeof(hdr)); 148 if (bytes != sizeof(hdr)) { 149 dev_dbg(&vdpasim->vdpa.dev, "request out header too short\n"); 150 goto err; 151 } 152 153 to_pull -= bytes; 154 155 type = vdpasim32_to_cpu(vdpasim, hdr.type); 156 sector = vdpasim64_to_cpu(vdpasim, hdr.sector); 157 offset = sector << SECTOR_SHIFT; 158 status = VIRTIO_BLK_S_OK; 159 160 if (type != VIRTIO_BLK_T_IN && type != VIRTIO_BLK_T_OUT && 161 sector != 0) { 162 dev_dbg(&vdpasim->vdpa.dev, 163 "sector must be 0 for %u request - sector: 0x%llx\n", 164 type, sector); 165 status = VIRTIO_BLK_S_IOERR; 166 goto err_status; 167 } 168 169 switch (type) { 170 case VIRTIO_BLK_T_IN: 171 if (!vdpasim_blk_check_range(vdpasim, sector, 172 to_push >> SECTOR_SHIFT, 173 VDPASIM_BLK_SIZE_MAX * VDPASIM_BLK_SEG_MAX)) { 174 status = VIRTIO_BLK_S_IOERR; 175 break; 176 } 177 178 vdpasim_blk_buffer_lock(blk); 179 bytes = vringh_iov_push_iotlb(&vq->vring, &vq->in_iov, 180 blk->buffer + offset, to_push); 181 vdpasim_blk_buffer_unlock(blk); 182 if (bytes < 0) { 183 dev_dbg(&vdpasim->vdpa.dev, 184 "vringh_iov_push_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n", 185 bytes, offset, to_push); 186 status = VIRTIO_BLK_S_IOERR; 187 break; 188 } 189 190 pushed += bytes; 191 break; 192 193 case VIRTIO_BLK_T_OUT: 194 if (!vdpasim_blk_check_range(vdpasim, sector, 195 to_pull >> SECTOR_SHIFT, 196 VDPASIM_BLK_SIZE_MAX * VDPASIM_BLK_SEG_MAX)) { 197 status = VIRTIO_BLK_S_IOERR; 198 break; 199 } 200 201 vdpasim_blk_buffer_lock(blk); 202 bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov, 203 blk->buffer + offset, to_pull); 204 vdpasim_blk_buffer_unlock(blk); 205 if (bytes < 0) { 206 dev_dbg(&vdpasim->vdpa.dev, 207 "vringh_iov_pull_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n", 208 bytes, offset, to_pull); 209 status = VIRTIO_BLK_S_IOERR; 210 break; 211 } 212 break; 213 214 case VIRTIO_BLK_T_GET_ID: 215 bytes = vringh_iov_push_iotlb(&vq->vring, &vq->in_iov, 216 vdpasim_blk_id, 217 VIRTIO_BLK_ID_BYTES); 218 if (bytes < 0) { 219 dev_dbg(&vdpasim->vdpa.dev, 220 "vringh_iov_push_iotlb() error: %zd\n", bytes); 221 status = VIRTIO_BLK_S_IOERR; 222 break; 223 } 224 225 pushed += bytes; 226 break; 227 228 case VIRTIO_BLK_T_FLUSH: 229 /* nothing to do */ 230 break; 231 232 case VIRTIO_BLK_T_DISCARD: 233 case VIRTIO_BLK_T_WRITE_ZEROES: { 234 struct virtio_blk_discard_write_zeroes range; 235 u32 num_sectors, flags; 236 237 if (to_pull != sizeof(range)) { 238 dev_dbg(&vdpasim->vdpa.dev, 239 "discard/write_zeroes header len: 0x%zx [expected: 0x%zx]\n", 240 to_pull, sizeof(range)); 241 status = VIRTIO_BLK_S_IOERR; 242 break; 243 } 244 245 bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov, &range, 246 to_pull); 247 if (bytes < 0) { 248 dev_dbg(&vdpasim->vdpa.dev, 249 "vringh_iov_pull_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n", 250 bytes, offset, to_pull); 251 status = VIRTIO_BLK_S_IOERR; 252 break; 253 } 254 255 sector = le64_to_cpu(range.sector); 256 offset = sector << SECTOR_SHIFT; 257 num_sectors = le32_to_cpu(range.num_sectors); 258 flags = le32_to_cpu(range.flags); 259 260 if (type == VIRTIO_BLK_T_DISCARD && flags != 0) { 261 dev_dbg(&vdpasim->vdpa.dev, 262 "discard unexpected flags set - flags: 0x%x\n", 263 flags); 264 status = VIRTIO_BLK_S_UNSUPP; 265 break; 266 } 267 268 if (type == VIRTIO_BLK_T_WRITE_ZEROES && 269 flags & ~VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP) { 270 dev_dbg(&vdpasim->vdpa.dev, 271 "write_zeroes unexpected flags set - flags: 0x%x\n", 272 flags); 273 status = VIRTIO_BLK_S_UNSUPP; 274 break; 275 } 276 277 if (!vdpasim_blk_check_range(vdpasim, sector, num_sectors, 278 VDPASIM_BLK_DWZ_MAX_SECTORS)) { 279 status = VIRTIO_BLK_S_IOERR; 280 break; 281 } 282 283 if (type == VIRTIO_BLK_T_WRITE_ZEROES) { 284 vdpasim_blk_buffer_lock(blk); 285 memset(blk->buffer + offset, 0, 286 num_sectors << SECTOR_SHIFT); 287 vdpasim_blk_buffer_unlock(blk); 288 } 289 290 break; 291 } 292 default: 293 dev_dbg(&vdpasim->vdpa.dev, 294 "Unsupported request type %d\n", type); 295 status = VIRTIO_BLK_S_IOERR; 296 break; 297 } 298 299 err_status: 300 /* If some operations fail, we need to skip the remaining bytes 301 * to put the status in the last byte 302 */ 303 if (to_push - pushed > 0) 304 vringh_kiov_advance(&vq->in_iov, to_push - pushed); 305 306 /* Last byte is the status */ 307 bytes = vringh_iov_push_iotlb(&vq->vring, &vq->in_iov, &status, 1); 308 if (bytes != 1) 309 goto err; 310 311 pushed += bytes; 312 313 /* Make sure data is wrote before advancing index */ 314 smp_wmb(); 315 316 handled = true; 317 318 err: 319 vringh_complete_iotlb(&vq->vring, vq->head, pushed); 320 321 return handled; 322 } 323 324 static void vdpasim_blk_work(struct vdpasim *vdpasim) 325 { 326 bool reschedule = false; 327 int i; 328 329 mutex_lock(&vdpasim->mutex); 330 331 if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK)) 332 goto out; 333 334 if (!vdpasim->running) 335 goto out; 336 337 for (i = 0; i < VDPASIM_BLK_VQ_NUM; i++) { 338 struct vdpasim_virtqueue *vq = &vdpasim->vqs[i]; 339 int reqs = 0; 340 341 if (!vq->ready) 342 continue; 343 344 while (vdpasim_blk_handle_req(vdpasim, vq)) { 345 /* Make sure used is visible before rasing the interrupt. */ 346 smp_wmb(); 347 348 local_bh_disable(); 349 if (vringh_need_notify_iotlb(&vq->vring) > 0) 350 vringh_notify(&vq->vring); 351 local_bh_enable(); 352 353 if (++reqs > 4) { 354 reschedule = true; 355 break; 356 } 357 } 358 } 359 out: 360 mutex_unlock(&vdpasim->mutex); 361 362 if (reschedule) 363 vdpasim_schedule_work(vdpasim); 364 } 365 366 static void vdpasim_blk_get_config(struct vdpasim *vdpasim, void *config) 367 { 368 struct virtio_blk_config *blk_config = config; 369 370 memset(config, 0, sizeof(struct virtio_blk_config)); 371 372 blk_config->capacity = cpu_to_vdpasim64(vdpasim, VDPASIM_BLK_CAPACITY); 373 blk_config->size_max = cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_SIZE_MAX); 374 blk_config->seg_max = cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_SEG_MAX); 375 blk_config->num_queues = cpu_to_vdpasim16(vdpasim, VDPASIM_BLK_VQ_NUM); 376 blk_config->min_io_size = cpu_to_vdpasim16(vdpasim, 1); 377 blk_config->opt_io_size = cpu_to_vdpasim32(vdpasim, 1); 378 blk_config->blk_size = cpu_to_vdpasim32(vdpasim, SECTOR_SIZE); 379 /* VIRTIO_BLK_F_DISCARD */ 380 blk_config->discard_sector_alignment = 381 cpu_to_vdpasim32(vdpasim, SECTOR_SIZE); 382 blk_config->max_discard_sectors = 383 cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_DWZ_MAX_SECTORS); 384 blk_config->max_discard_seg = cpu_to_vdpasim32(vdpasim, 1); 385 /* VIRTIO_BLK_F_WRITE_ZEROES */ 386 blk_config->max_write_zeroes_sectors = 387 cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_DWZ_MAX_SECTORS); 388 blk_config->max_write_zeroes_seg = cpu_to_vdpasim32(vdpasim, 1); 389 390 } 391 392 static void vdpasim_blk_free(struct vdpasim *vdpasim) 393 { 394 struct vdpasim_blk *blk = sim_to_blk(vdpasim); 395 396 if (!blk->shared_backend) 397 kvfree(blk->buffer); 398 } 399 400 static void vdpasim_blk_mgmtdev_release(struct device *dev) 401 { 402 } 403 404 static struct device vdpasim_blk_mgmtdev = { 405 .init_name = "vdpasim_blk", 406 .release = vdpasim_blk_mgmtdev_release, 407 }; 408 409 static int vdpasim_blk_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, 410 const struct vdpa_dev_set_config *config) 411 { 412 struct vdpasim_dev_attr dev_attr = {}; 413 struct vdpasim_blk *blk; 414 struct vdpasim *simdev; 415 int ret; 416 417 dev_attr.mgmt_dev = mdev; 418 dev_attr.name = name; 419 dev_attr.id = VIRTIO_ID_BLOCK; 420 dev_attr.supported_features = VDPASIM_BLK_FEATURES; 421 dev_attr.nvqs = VDPASIM_BLK_VQ_NUM; 422 dev_attr.ngroups = VDPASIM_BLK_GROUP_NUM; 423 dev_attr.nas = VDPASIM_BLK_AS_NUM; 424 dev_attr.alloc_size = sizeof(struct vdpasim_blk); 425 dev_attr.config_size = sizeof(struct virtio_blk_config); 426 dev_attr.get_config = vdpasim_blk_get_config; 427 dev_attr.work_fn = vdpasim_blk_work; 428 dev_attr.free = vdpasim_blk_free; 429 430 simdev = vdpasim_create(&dev_attr, config); 431 if (IS_ERR(simdev)) 432 return PTR_ERR(simdev); 433 434 blk = sim_to_blk(simdev); 435 blk->shared_backend = shared_backend; 436 437 if (blk->shared_backend) { 438 blk->buffer = shared_buffer; 439 } else { 440 blk->buffer = kvmalloc(VDPASIM_BLK_CAPACITY << SECTOR_SHIFT, 441 GFP_KERNEL); 442 if (!blk->buffer) { 443 ret = -ENOMEM; 444 goto put_dev; 445 } 446 } 447 448 ret = _vdpa_register_device(&simdev->vdpa, VDPASIM_BLK_VQ_NUM); 449 if (ret) 450 goto put_dev; 451 452 return 0; 453 454 put_dev: 455 put_device(&simdev->vdpa.dev); 456 return ret; 457 } 458 459 static void vdpasim_blk_dev_del(struct vdpa_mgmt_dev *mdev, 460 struct vdpa_device *dev) 461 { 462 struct vdpasim *simdev = container_of(dev, struct vdpasim, vdpa); 463 464 _vdpa_unregister_device(&simdev->vdpa); 465 } 466 467 static const struct vdpa_mgmtdev_ops vdpasim_blk_mgmtdev_ops = { 468 .dev_add = vdpasim_blk_dev_add, 469 .dev_del = vdpasim_blk_dev_del 470 }; 471 472 static struct virtio_device_id id_table[] = { 473 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID }, 474 { 0 }, 475 }; 476 477 static struct vdpa_mgmt_dev mgmt_dev = { 478 .device = &vdpasim_blk_mgmtdev, 479 .id_table = id_table, 480 .ops = &vdpasim_blk_mgmtdev_ops, 481 }; 482 483 static int __init vdpasim_blk_init(void) 484 { 485 int ret; 486 487 ret = device_register(&vdpasim_blk_mgmtdev); 488 if (ret) { 489 put_device(&vdpasim_blk_mgmtdev); 490 return ret; 491 } 492 493 ret = vdpa_mgmtdev_register(&mgmt_dev); 494 if (ret) 495 goto parent_err; 496 497 if (shared_backend) { 498 shared_buffer = kvmalloc(VDPASIM_BLK_CAPACITY << SECTOR_SHIFT, 499 GFP_KERNEL); 500 if (!shared_buffer) { 501 ret = -ENOMEM; 502 goto parent_err; 503 } 504 } 505 506 return 0; 507 508 parent_err: 509 device_unregister(&vdpasim_blk_mgmtdev); 510 return ret; 511 } 512 513 static void __exit vdpasim_blk_exit(void) 514 { 515 kvfree(shared_buffer); 516 vdpa_mgmtdev_unregister(&mgmt_dev); 517 device_unregister(&vdpasim_blk_mgmtdev); 518 } 519 520 module_init(vdpasim_blk_init) 521 module_exit(vdpasim_blk_exit) 522 523 MODULE_VERSION(DRV_VERSION); 524 MODULE_LICENSE(DRV_LICENSE); 525 MODULE_AUTHOR(DRV_AUTHOR); 526 MODULE_DESCRIPTION(DRV_DESC); 527