1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * VDPA simulator for block device.
4 *
5 * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
6 * Copyright (c) 2021, Red Hat Inc. All rights reserved.
7 *
8 */
9
10 #include <linux/init.h>
11 #include <linux/module.h>
12 #include <linux/device.h>
13 #include <linux/kernel.h>
14 #include <linux/blkdev.h>
15 #include <linux/vringh.h>
16 #include <linux/vdpa.h>
17 #include <uapi/linux/virtio_blk.h>
18
19 #include "vdpa_sim.h"
20
21 #define DRV_VERSION "0.1"
22 #define DRV_AUTHOR "Max Gurtovoy <mgurtovoy@nvidia.com>"
23 #define DRV_DESC "vDPA Device Simulator for block device"
24 #define DRV_LICENSE "GPL v2"
25
26 #define VDPASIM_BLK_FEATURES (VDPASIM_FEATURES | \
27 (1ULL << VIRTIO_BLK_F_FLUSH) | \
28 (1ULL << VIRTIO_BLK_F_SIZE_MAX) | \
29 (1ULL << VIRTIO_BLK_F_SEG_MAX) | \
30 (1ULL << VIRTIO_BLK_F_BLK_SIZE) | \
31 (1ULL << VIRTIO_BLK_F_TOPOLOGY) | \
32 (1ULL << VIRTIO_BLK_F_MQ) | \
33 (1ULL << VIRTIO_BLK_F_DISCARD) | \
34 (1ULL << VIRTIO_BLK_F_WRITE_ZEROES))
35
36 #define VDPASIM_BLK_CAPACITY 0x40000
37 #define VDPASIM_BLK_SIZE_MAX 0x1000
38 #define VDPASIM_BLK_SEG_MAX 32
39 #define VDPASIM_BLK_DWZ_MAX_SECTORS UINT_MAX
40
41 /* 1 virtqueue, 1 address space, 1 virtqueue group */
42 #define VDPASIM_BLK_VQ_NUM 1
43 #define VDPASIM_BLK_AS_NUM 1
44 #define VDPASIM_BLK_GROUP_NUM 1
45
46 struct vdpasim_blk {
47 struct vdpasim vdpasim;
48 void *buffer;
49 bool shared_backend;
50 };
51
sim_to_blk(struct vdpasim * vdpasim)52 static struct vdpasim_blk *sim_to_blk(struct vdpasim *vdpasim)
53 {
54 return container_of(vdpasim, struct vdpasim_blk, vdpasim);
55 }
56
57 static char vdpasim_blk_id[VIRTIO_BLK_ID_BYTES] = "vdpa_blk_sim";
58
59 static bool shared_backend;
60 module_param(shared_backend, bool, 0444);
61 MODULE_PARM_DESC(shared_backend, "Enable the shared backend between virtio-blk devices");
62
63 static void *shared_buffer;
64 /* mutex to synchronize shared_buffer access */
65 static DEFINE_MUTEX(shared_buffer_mutex);
66
vdpasim_blk_buffer_lock(struct vdpasim_blk * blk)67 static void vdpasim_blk_buffer_lock(struct vdpasim_blk *blk)
68 {
69 if (blk->shared_backend)
70 mutex_lock(&shared_buffer_mutex);
71 }
72
vdpasim_blk_buffer_unlock(struct vdpasim_blk * blk)73 static void vdpasim_blk_buffer_unlock(struct vdpasim_blk *blk)
74 {
75 if (blk->shared_backend)
76 mutex_unlock(&shared_buffer_mutex);
77 }
78
vdpasim_blk_check_range(struct vdpasim * vdpasim,u64 start_sector,u64 num_sectors,u64 max_sectors)79 static bool vdpasim_blk_check_range(struct vdpasim *vdpasim, u64 start_sector,
80 u64 num_sectors, u64 max_sectors)
81 {
82 if (start_sector > VDPASIM_BLK_CAPACITY) {
83 dev_dbg(&vdpasim->vdpa.dev,
84 "starting sector exceeds the capacity - start: 0x%llx capacity: 0x%x\n",
85 start_sector, VDPASIM_BLK_CAPACITY);
86 }
87
88 if (num_sectors > max_sectors) {
89 dev_dbg(&vdpasim->vdpa.dev,
90 "number of sectors exceeds the max allowed in a request - num: 0x%llx max: 0x%llx\n",
91 num_sectors, max_sectors);
92 return false;
93 }
94
95 if (num_sectors > VDPASIM_BLK_CAPACITY - start_sector) {
96 dev_dbg(&vdpasim->vdpa.dev,
97 "request exceeds the capacity - start: 0x%llx num: 0x%llx capacity: 0x%x\n",
98 start_sector, num_sectors, VDPASIM_BLK_CAPACITY);
99 return false;
100 }
101
102 return true;
103 }
104
105 /* Returns 'true' if the request is handled (with or without an I/O error)
106 * and the status is correctly written in the last byte of the 'in iov',
107 * 'false' otherwise.
108 */
vdpasim_blk_handle_req(struct vdpasim * vdpasim,struct vdpasim_virtqueue * vq)109 static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim,
110 struct vdpasim_virtqueue *vq)
111 {
112 struct vdpasim_blk *blk = sim_to_blk(vdpasim);
113 size_t pushed = 0, to_pull, to_push;
114 struct virtio_blk_outhdr hdr;
115 bool handled = false;
116 ssize_t bytes;
117 loff_t offset;
118 u64 sector;
119 u8 status;
120 u32 type;
121 int ret;
122
123 ret = vringh_getdesc_iotlb(&vq->vring, &vq->out_iov, &vq->in_iov,
124 &vq->head, GFP_ATOMIC);
125 if (ret != 1)
126 return false;
127
128 if (vq->out_iov.used < 1 || vq->in_iov.used < 1) {
129 dev_dbg(&vdpasim->vdpa.dev, "missing headers - out_iov: %u in_iov %u\n",
130 vq->out_iov.used, vq->in_iov.used);
131 goto err;
132 }
133
134 if (vq->in_iov.iov[vq->in_iov.used - 1].iov_len < 1) {
135 dev_dbg(&vdpasim->vdpa.dev, "request in header too short\n");
136 goto err;
137 }
138
139 /* The last byte is the status and we checked if the last iov has
140 * enough room for it.
141 */
142 to_push = vringh_kiov_length(&vq->in_iov) - 1;
143
144 to_pull = vringh_kiov_length(&vq->out_iov);
145
146 bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov, &hdr,
147 sizeof(hdr));
148 if (bytes != sizeof(hdr)) {
149 dev_dbg(&vdpasim->vdpa.dev, "request out header too short\n");
150 goto err;
151 }
152
153 to_pull -= bytes;
154
155 type = vdpasim32_to_cpu(vdpasim, hdr.type);
156 sector = vdpasim64_to_cpu(vdpasim, hdr.sector);
157 offset = sector << SECTOR_SHIFT;
158 status = VIRTIO_BLK_S_OK;
159
160 if (type != VIRTIO_BLK_T_IN && type != VIRTIO_BLK_T_OUT &&
161 sector != 0) {
162 dev_dbg(&vdpasim->vdpa.dev,
163 "sector must be 0 for %u request - sector: 0x%llx\n",
164 type, sector);
165 status = VIRTIO_BLK_S_IOERR;
166 goto err_status;
167 }
168
169 switch (type) {
170 case VIRTIO_BLK_T_IN:
171 if (!vdpasim_blk_check_range(vdpasim, sector,
172 to_push >> SECTOR_SHIFT,
173 VDPASIM_BLK_SIZE_MAX * VDPASIM_BLK_SEG_MAX)) {
174 status = VIRTIO_BLK_S_IOERR;
175 break;
176 }
177
178 vdpasim_blk_buffer_lock(blk);
179 bytes = vringh_iov_push_iotlb(&vq->vring, &vq->in_iov,
180 blk->buffer + offset, to_push);
181 vdpasim_blk_buffer_unlock(blk);
182 if (bytes < 0) {
183 dev_dbg(&vdpasim->vdpa.dev,
184 "vringh_iov_push_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
185 bytes, offset, to_push);
186 status = VIRTIO_BLK_S_IOERR;
187 break;
188 }
189
190 pushed += bytes;
191 break;
192
193 case VIRTIO_BLK_T_OUT:
194 if (!vdpasim_blk_check_range(vdpasim, sector,
195 to_pull >> SECTOR_SHIFT,
196 VDPASIM_BLK_SIZE_MAX * VDPASIM_BLK_SEG_MAX)) {
197 status = VIRTIO_BLK_S_IOERR;
198 break;
199 }
200
201 vdpasim_blk_buffer_lock(blk);
202 bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov,
203 blk->buffer + offset, to_pull);
204 vdpasim_blk_buffer_unlock(blk);
205 if (bytes < 0) {
206 dev_dbg(&vdpasim->vdpa.dev,
207 "vringh_iov_pull_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
208 bytes, offset, to_pull);
209 status = VIRTIO_BLK_S_IOERR;
210 break;
211 }
212 break;
213
214 case VIRTIO_BLK_T_GET_ID:
215 bytes = vringh_iov_push_iotlb(&vq->vring, &vq->in_iov,
216 vdpasim_blk_id,
217 VIRTIO_BLK_ID_BYTES);
218 if (bytes < 0) {
219 dev_dbg(&vdpasim->vdpa.dev,
220 "vringh_iov_push_iotlb() error: %zd\n", bytes);
221 status = VIRTIO_BLK_S_IOERR;
222 break;
223 }
224
225 pushed += bytes;
226 break;
227
228 case VIRTIO_BLK_T_FLUSH:
229 /* nothing to do */
230 break;
231
232 case VIRTIO_BLK_T_DISCARD:
233 case VIRTIO_BLK_T_WRITE_ZEROES: {
234 struct virtio_blk_discard_write_zeroes range;
235 u32 num_sectors, flags;
236
237 if (to_pull != sizeof(range)) {
238 dev_dbg(&vdpasim->vdpa.dev,
239 "discard/write_zeroes header len: 0x%zx [expected: 0x%zx]\n",
240 to_pull, sizeof(range));
241 status = VIRTIO_BLK_S_IOERR;
242 break;
243 }
244
245 bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov, &range,
246 to_pull);
247 if (bytes < 0) {
248 dev_dbg(&vdpasim->vdpa.dev,
249 "vringh_iov_pull_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
250 bytes, offset, to_pull);
251 status = VIRTIO_BLK_S_IOERR;
252 break;
253 }
254
255 sector = le64_to_cpu(range.sector);
256 offset = sector << SECTOR_SHIFT;
257 num_sectors = le32_to_cpu(range.num_sectors);
258 flags = le32_to_cpu(range.flags);
259
260 if (type == VIRTIO_BLK_T_DISCARD && flags != 0) {
261 dev_dbg(&vdpasim->vdpa.dev,
262 "discard unexpected flags set - flags: 0x%x\n",
263 flags);
264 status = VIRTIO_BLK_S_UNSUPP;
265 break;
266 }
267
268 if (type == VIRTIO_BLK_T_WRITE_ZEROES &&
269 flags & ~VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP) {
270 dev_dbg(&vdpasim->vdpa.dev,
271 "write_zeroes unexpected flags set - flags: 0x%x\n",
272 flags);
273 status = VIRTIO_BLK_S_UNSUPP;
274 break;
275 }
276
277 if (!vdpasim_blk_check_range(vdpasim, sector, num_sectors,
278 VDPASIM_BLK_DWZ_MAX_SECTORS)) {
279 status = VIRTIO_BLK_S_IOERR;
280 break;
281 }
282
283 if (type == VIRTIO_BLK_T_WRITE_ZEROES) {
284 vdpasim_blk_buffer_lock(blk);
285 memset(blk->buffer + offset, 0,
286 num_sectors << SECTOR_SHIFT);
287 vdpasim_blk_buffer_unlock(blk);
288 }
289
290 break;
291 }
292 default:
293 dev_dbg(&vdpasim->vdpa.dev,
294 "Unsupported request type %d\n", type);
295 status = VIRTIO_BLK_S_IOERR;
296 break;
297 }
298
299 err_status:
300 /* If some operations fail, we need to skip the remaining bytes
301 * to put the status in the last byte
302 */
303 if (to_push - pushed > 0)
304 vringh_kiov_advance(&vq->in_iov, to_push - pushed);
305
306 /* Last byte is the status */
307 bytes = vringh_iov_push_iotlb(&vq->vring, &vq->in_iov, &status, 1);
308 if (bytes != 1)
309 goto err;
310
311 pushed += bytes;
312
313 /* Make sure data is wrote before advancing index */
314 smp_wmb();
315
316 handled = true;
317
318 err:
319 vringh_complete_iotlb(&vq->vring, vq->head, pushed);
320
321 return handled;
322 }
323
vdpasim_blk_work(struct vdpasim * vdpasim)324 static void vdpasim_blk_work(struct vdpasim *vdpasim)
325 {
326 bool reschedule = false;
327 int i;
328
329 mutex_lock(&vdpasim->mutex);
330
331 if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK))
332 goto out;
333
334 if (!vdpasim->running)
335 goto out;
336
337 for (i = 0; i < VDPASIM_BLK_VQ_NUM; i++) {
338 struct vdpasim_virtqueue *vq = &vdpasim->vqs[i];
339 int reqs = 0;
340
341 if (!vq->ready)
342 continue;
343
344 while (vdpasim_blk_handle_req(vdpasim, vq)) {
345 /* Make sure used is visible before rasing the interrupt. */
346 smp_wmb();
347
348 local_bh_disable();
349 if (vringh_need_notify_iotlb(&vq->vring) > 0)
350 vringh_notify(&vq->vring);
351 local_bh_enable();
352
353 if (++reqs > 4) {
354 reschedule = true;
355 break;
356 }
357 }
358 }
359 out:
360 mutex_unlock(&vdpasim->mutex);
361
362 if (reschedule)
363 vdpasim_schedule_work(vdpasim);
364 }
365
vdpasim_blk_get_config(struct vdpasim * vdpasim,void * config)366 static void vdpasim_blk_get_config(struct vdpasim *vdpasim, void *config)
367 {
368 struct virtio_blk_config *blk_config = config;
369
370 memset(config, 0, sizeof(struct virtio_blk_config));
371
372 blk_config->capacity = cpu_to_vdpasim64(vdpasim, VDPASIM_BLK_CAPACITY);
373 blk_config->size_max = cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_SIZE_MAX);
374 blk_config->seg_max = cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_SEG_MAX);
375 blk_config->num_queues = cpu_to_vdpasim16(vdpasim, VDPASIM_BLK_VQ_NUM);
376 blk_config->min_io_size = cpu_to_vdpasim16(vdpasim, 1);
377 blk_config->opt_io_size = cpu_to_vdpasim32(vdpasim, 1);
378 blk_config->blk_size = cpu_to_vdpasim32(vdpasim, SECTOR_SIZE);
379 /* VIRTIO_BLK_F_DISCARD */
380 blk_config->discard_sector_alignment =
381 cpu_to_vdpasim32(vdpasim, SECTOR_SIZE);
382 blk_config->max_discard_sectors =
383 cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_DWZ_MAX_SECTORS);
384 blk_config->max_discard_seg = cpu_to_vdpasim32(vdpasim, 1);
385 /* VIRTIO_BLK_F_WRITE_ZEROES */
386 blk_config->max_write_zeroes_sectors =
387 cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_DWZ_MAX_SECTORS);
388 blk_config->max_write_zeroes_seg = cpu_to_vdpasim32(vdpasim, 1);
389
390 }
391
vdpasim_blk_free(struct vdpasim * vdpasim)392 static void vdpasim_blk_free(struct vdpasim *vdpasim)
393 {
394 struct vdpasim_blk *blk = sim_to_blk(vdpasim);
395
396 if (!blk->shared_backend)
397 kvfree(blk->buffer);
398 }
399
vdpasim_blk_mgmtdev_release(struct device * dev)400 static void vdpasim_blk_mgmtdev_release(struct device *dev)
401 {
402 }
403
404 static struct device vdpasim_blk_mgmtdev = {
405 .init_name = "vdpasim_blk",
406 .release = vdpasim_blk_mgmtdev_release,
407 };
408
vdpasim_blk_dev_add(struct vdpa_mgmt_dev * mdev,const char * name,const struct vdpa_dev_set_config * config)409 static int vdpasim_blk_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
410 const struct vdpa_dev_set_config *config)
411 {
412 struct vdpasim_dev_attr dev_attr = {};
413 struct vdpasim_blk *blk;
414 struct vdpasim *simdev;
415 int ret;
416
417 dev_attr.mgmt_dev = mdev;
418 dev_attr.name = name;
419 dev_attr.id = VIRTIO_ID_BLOCK;
420 dev_attr.supported_features = VDPASIM_BLK_FEATURES;
421 dev_attr.nvqs = VDPASIM_BLK_VQ_NUM;
422 dev_attr.ngroups = VDPASIM_BLK_GROUP_NUM;
423 dev_attr.nas = VDPASIM_BLK_AS_NUM;
424 dev_attr.alloc_size = sizeof(struct vdpasim_blk);
425 dev_attr.config_size = sizeof(struct virtio_blk_config);
426 dev_attr.get_config = vdpasim_blk_get_config;
427 dev_attr.work_fn = vdpasim_blk_work;
428 dev_attr.free = vdpasim_blk_free;
429
430 simdev = vdpasim_create(&dev_attr, config);
431 if (IS_ERR(simdev))
432 return PTR_ERR(simdev);
433
434 blk = sim_to_blk(simdev);
435 blk->shared_backend = shared_backend;
436
437 if (blk->shared_backend) {
438 blk->buffer = shared_buffer;
439 } else {
440 blk->buffer = kvzalloc(VDPASIM_BLK_CAPACITY << SECTOR_SHIFT,
441 GFP_KERNEL);
442 if (!blk->buffer) {
443 ret = -ENOMEM;
444 goto put_dev;
445 }
446 }
447
448 ret = _vdpa_register_device(&simdev->vdpa, VDPASIM_BLK_VQ_NUM);
449 if (ret)
450 goto put_dev;
451
452 return 0;
453
454 put_dev:
455 put_device(&simdev->vdpa.dev);
456 return ret;
457 }
458
vdpasim_blk_dev_del(struct vdpa_mgmt_dev * mdev,struct vdpa_device * dev)459 static void vdpasim_blk_dev_del(struct vdpa_mgmt_dev *mdev,
460 struct vdpa_device *dev)
461 {
462 struct vdpasim *simdev = container_of(dev, struct vdpasim, vdpa);
463
464 _vdpa_unregister_device(&simdev->vdpa);
465 }
466
467 static const struct vdpa_mgmtdev_ops vdpasim_blk_mgmtdev_ops = {
468 .dev_add = vdpasim_blk_dev_add,
469 .dev_del = vdpasim_blk_dev_del
470 };
471
472 static struct virtio_device_id id_table[] = {
473 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID },
474 { 0 },
475 };
476
477 static struct vdpa_mgmt_dev mgmt_dev = {
478 .device = &vdpasim_blk_mgmtdev,
479 .id_table = id_table,
480 .ops = &vdpasim_blk_mgmtdev_ops,
481 };
482
vdpasim_blk_init(void)483 static int __init vdpasim_blk_init(void)
484 {
485 int ret;
486
487 ret = device_register(&vdpasim_blk_mgmtdev);
488 if (ret) {
489 put_device(&vdpasim_blk_mgmtdev);
490 return ret;
491 }
492
493 ret = vdpa_mgmtdev_register(&mgmt_dev);
494 if (ret)
495 goto parent_err;
496
497 if (shared_backend) {
498 shared_buffer = kvzalloc(VDPASIM_BLK_CAPACITY << SECTOR_SHIFT,
499 GFP_KERNEL);
500 if (!shared_buffer) {
501 ret = -ENOMEM;
502 goto mgmt_dev_err;
503 }
504 }
505
506 return 0;
507 mgmt_dev_err:
508 vdpa_mgmtdev_unregister(&mgmt_dev);
509 parent_err:
510 device_unregister(&vdpasim_blk_mgmtdev);
511 return ret;
512 }
513
vdpasim_blk_exit(void)514 static void __exit vdpasim_blk_exit(void)
515 {
516 kvfree(shared_buffer);
517 vdpa_mgmtdev_unregister(&mgmt_dev);
518 device_unregister(&vdpasim_blk_mgmtdev);
519 }
520
521 module_init(vdpasim_blk_init)
522 module_exit(vdpasim_blk_exit)
523
524 MODULE_VERSION(DRV_VERSION);
525 MODULE_LICENSE(DRV_LICENSE);
526 MODULE_AUTHOR(DRV_AUTHOR);
527 MODULE_DESCRIPTION(DRV_DESC);
528