1 // SPDX-License-Identifier: GPL-2.0 2 #define _GNU_SOURCE 3 #include <getopt.h> 4 #include <limits.h> 5 #include <string.h> 6 #include <poll.h> 7 #include <sys/eventfd.h> 8 #include <stdlib.h> 9 #include <assert.h> 10 #include <unistd.h> 11 #include <sys/ioctl.h> 12 #include <sys/stat.h> 13 #include <sys/types.h> 14 #include <fcntl.h> 15 #include <stdbool.h> 16 #include <linux/virtio_types.h> 17 #include <linux/vhost.h> 18 #include <linux/virtio.h> 19 #include <linux/virtio_ring.h> 20 #include "../../drivers/vhost/test.h" 21 22 #define RANDOM_BATCH -1 23 24 /* Unused */ 25 void *__kmalloc_fake, *__kfree_ignore_start, *__kfree_ignore_end; 26 27 struct vq_info { 28 int kick; 29 int call; 30 int num; 31 int idx; 32 void *ring; 33 /* copy used for control */ 34 struct vring vring; 35 struct virtqueue *vq; 36 }; 37 38 struct vdev_info { 39 struct virtio_device vdev; 40 int control; 41 struct pollfd fds[1]; 42 struct vq_info vqs[1]; 43 int nvqs; 44 void *buf; 45 size_t buf_size; 46 struct vhost_memory *mem; 47 }; 48 49 static const struct vhost_vring_file no_backend = { .fd = -1 }, 50 backend = { .fd = 1 }; 51 static const struct vhost_vring_state null_state = {}; 52 53 bool vq_notify(struct virtqueue *vq) 54 { 55 struct vq_info *info = vq->priv; 56 unsigned long long v = 1; 57 int r; 58 r = write(info->kick, &v, sizeof v); 59 assert(r == sizeof v); 60 return true; 61 } 62 63 void vq_callback(struct virtqueue *vq) 64 { 65 } 66 67 68 void vhost_vq_setup(struct vdev_info *dev, struct vq_info *info) 69 { 70 struct vhost_vring_state state = { .index = info->idx }; 71 struct vhost_vring_file file = { .index = info->idx }; 72 unsigned long long features = dev->vdev.features; 73 struct vhost_vring_addr addr = { 74 .index = info->idx, 75 .desc_user_addr = (uint64_t)(unsigned long)info->vring.desc, 76 .avail_user_addr = (uint64_t)(unsigned long)info->vring.avail, 77 .used_user_addr = (uint64_t)(unsigned long)info->vring.used, 78 }; 79 int r; 80 r = ioctl(dev->control, VHOST_SET_FEATURES, &features); 81 assert(r >= 0); 82 state.num = info->vring.num; 83 r = ioctl(dev->control, VHOST_SET_VRING_NUM, &state); 84 assert(r >= 0); 85 state.num = 0; 86 r = ioctl(dev->control, VHOST_SET_VRING_BASE, &state); 87 assert(r >= 0); 88 r = ioctl(dev->control, VHOST_SET_VRING_ADDR, &addr); 89 assert(r >= 0); 90 file.fd = info->kick; 91 r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file); 92 assert(r >= 0); 93 file.fd = info->call; 94 r = ioctl(dev->control, VHOST_SET_VRING_CALL, &file); 95 assert(r >= 0); 96 } 97 98 static void vq_reset(struct vq_info *info, int num, struct virtio_device *vdev) 99 { 100 if (info->vq) 101 vring_del_virtqueue(info->vq); 102 103 memset(info->ring, 0, vring_size(num, 4096)); 104 vring_init(&info->vring, num, info->ring, 4096); 105 info->vq = vring_new_virtqueue(info->idx, num, 4096, vdev, true, false, 106 info->ring, vq_notify, vq_callback, "test"); 107 assert(info->vq); 108 info->vq->priv = info; 109 } 110 111 static void vq_info_add(struct vdev_info *dev, int num) 112 { 113 struct vq_info *info = &dev->vqs[dev->nvqs]; 114 int r; 115 info->idx = dev->nvqs; 116 info->kick = eventfd(0, EFD_NONBLOCK); 117 info->call = eventfd(0, EFD_NONBLOCK); 118 r = posix_memalign(&info->ring, 4096, vring_size(num, 4096)); 119 assert(r >= 0); 120 vq_reset(info, num, &dev->vdev); 121 vhost_vq_setup(dev, info); 122 dev->fds[info->idx].fd = info->call; 123 dev->fds[info->idx].events = POLLIN; 124 dev->nvqs++; 125 } 126 127 static void vdev_info_init(struct vdev_info* dev, unsigned long long features) 128 { 129 int r; 130 memset(dev, 0, sizeof *dev); 131 dev->vdev.features = features; 132 INIT_LIST_HEAD(&dev->vdev.vqs); 133 spin_lock_init(&dev->vdev.vqs_list_lock); 134 dev->buf_size = 1024; 135 dev->buf = malloc(dev->buf_size); 136 assert(dev->buf); 137 dev->control = open("/dev/vhost-test", O_RDWR); 138 assert(dev->control >= 0); 139 r = ioctl(dev->control, VHOST_SET_OWNER, NULL); 140 assert(r >= 0); 141 dev->mem = malloc(offsetof(struct vhost_memory, regions) + 142 sizeof dev->mem->regions[0]); 143 assert(dev->mem); 144 memset(dev->mem, 0, offsetof(struct vhost_memory, regions) + 145 sizeof dev->mem->regions[0]); 146 dev->mem->nregions = 1; 147 dev->mem->regions[0].guest_phys_addr = (long)dev->buf; 148 dev->mem->regions[0].userspace_addr = (long)dev->buf; 149 dev->mem->regions[0].memory_size = dev->buf_size; 150 r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem); 151 assert(r >= 0); 152 } 153 154 /* TODO: this is pretty bad: we get a cache line bounce 155 * for the wait queue on poll and another one on read, 156 * plus the read which is there just to clear the 157 * current state. */ 158 static void wait_for_interrupt(struct vdev_info *dev) 159 { 160 int i; 161 unsigned long long val; 162 poll(dev->fds, dev->nvqs, -1); 163 for (i = 0; i < dev->nvqs; ++i) 164 if (dev->fds[i].revents & POLLIN) { 165 read(dev->fds[i].fd, &val, sizeof val); 166 } 167 } 168 169 static void run_test(struct vdev_info *dev, struct vq_info *vq, 170 bool delayed, int batch, int reset_n, int bufs) 171 { 172 struct scatterlist sl; 173 long started = 0, completed = 0, next_reset = reset_n; 174 long completed_before, started_before; 175 int r, test = 1; 176 unsigned int len; 177 long long spurious = 0; 178 const bool random_batch = batch == RANDOM_BATCH; 179 180 r = ioctl(dev->control, VHOST_TEST_RUN, &test); 181 assert(r >= 0); 182 if (!reset_n) { 183 next_reset = INT_MAX; 184 } 185 186 for (;;) { 187 virtqueue_disable_cb(vq->vq); 188 completed_before = completed; 189 started_before = started; 190 do { 191 const bool reset = completed > next_reset; 192 if (random_batch) 193 batch = (random() % vq->vring.num) + 1; 194 195 while (started < bufs && 196 (started - completed) < batch) { 197 sg_init_one(&sl, dev->buf, dev->buf_size); 198 r = virtqueue_add_outbuf(vq->vq, &sl, 1, 199 dev->buf + started, 200 GFP_ATOMIC); 201 if (unlikely(r != 0)) { 202 if (r == -ENOSPC && 203 started > started_before) 204 r = 0; 205 else 206 r = -1; 207 break; 208 } 209 210 ++started; 211 212 if (unlikely(!virtqueue_kick(vq->vq))) { 213 r = -1; 214 break; 215 } 216 } 217 218 if (started >= bufs) 219 r = -1; 220 221 if (reset) { 222 r = ioctl(dev->control, VHOST_TEST_SET_BACKEND, 223 &no_backend); 224 assert(!r); 225 } 226 227 /* Flush out completed bufs if any */ 228 while (virtqueue_get_buf(vq->vq, &len)) { 229 ++completed; 230 r = 0; 231 } 232 233 if (reset) { 234 struct vhost_vring_state s = { .index = 0 }; 235 236 vq_reset(vq, vq->vring.num, &dev->vdev); 237 238 r = ioctl(dev->control, VHOST_GET_VRING_BASE, 239 &s); 240 assert(!r); 241 242 s.num = 0; 243 r = ioctl(dev->control, VHOST_SET_VRING_BASE, 244 &null_state); 245 assert(!r); 246 247 r = ioctl(dev->control, VHOST_TEST_SET_BACKEND, 248 &backend); 249 assert(!r); 250 251 started = completed; 252 while (completed > next_reset) 253 next_reset += completed; 254 } 255 } while (r == 0); 256 if (completed == completed_before && started == started_before) 257 ++spurious; 258 assert(completed <= bufs); 259 assert(started <= bufs); 260 if (completed == bufs) 261 break; 262 if (delayed) { 263 if (virtqueue_enable_cb_delayed(vq->vq)) 264 wait_for_interrupt(dev); 265 } else { 266 if (virtqueue_enable_cb(vq->vq)) 267 wait_for_interrupt(dev); 268 } 269 } 270 test = 0; 271 r = ioctl(dev->control, VHOST_TEST_RUN, &test); 272 assert(r >= 0); 273 fprintf(stderr, 274 "spurious wakeups: 0x%llx started=0x%lx completed=0x%lx\n", 275 spurious, started, completed); 276 } 277 278 const char optstring[] = "h"; 279 const struct option longopts[] = { 280 { 281 .name = "help", 282 .val = 'h', 283 }, 284 { 285 .name = "event-idx", 286 .val = 'E', 287 }, 288 { 289 .name = "no-event-idx", 290 .val = 'e', 291 }, 292 { 293 .name = "indirect", 294 .val = 'I', 295 }, 296 { 297 .name = "no-indirect", 298 .val = 'i', 299 }, 300 { 301 .name = "virtio-1", 302 .val = '1', 303 }, 304 { 305 .name = "no-virtio-1", 306 .val = '0', 307 }, 308 { 309 .name = "delayed-interrupt", 310 .val = 'D', 311 }, 312 { 313 .name = "no-delayed-interrupt", 314 .val = 'd', 315 }, 316 { 317 .name = "batch", 318 .val = 'b', 319 .has_arg = required_argument, 320 }, 321 { 322 .name = "reset", 323 .val = 'r', 324 .has_arg = optional_argument, 325 }, 326 { 327 } 328 }; 329 330 static void help(void) 331 { 332 fprintf(stderr, "Usage: virtio_test [--help]" 333 " [--no-indirect]" 334 " [--no-event-idx]" 335 " [--no-virtio-1]" 336 " [--delayed-interrupt]" 337 " [--batch=random/N]" 338 " [--reset=N]" 339 "\n"); 340 } 341 342 int main(int argc, char **argv) 343 { 344 struct vdev_info dev; 345 unsigned long long features = (1ULL << VIRTIO_RING_F_INDIRECT_DESC) | 346 (1ULL << VIRTIO_RING_F_EVENT_IDX) | (1ULL << VIRTIO_F_VERSION_1); 347 long batch = 1, reset = 0; 348 int o; 349 bool delayed = false; 350 351 for (;;) { 352 o = getopt_long(argc, argv, optstring, longopts, NULL); 353 switch (o) { 354 case -1: 355 goto done; 356 case '?': 357 help(); 358 exit(2); 359 case 'e': 360 features &= ~(1ULL << VIRTIO_RING_F_EVENT_IDX); 361 break; 362 case 'h': 363 help(); 364 goto done; 365 case 'i': 366 features &= ~(1ULL << VIRTIO_RING_F_INDIRECT_DESC); 367 break; 368 case '0': 369 features &= ~(1ULL << VIRTIO_F_VERSION_1); 370 break; 371 case 'D': 372 delayed = true; 373 break; 374 case 'b': 375 if (0 == strcmp(optarg, "random")) { 376 batch = RANDOM_BATCH; 377 } else { 378 batch = strtol(optarg, NULL, 10); 379 assert(batch > 0); 380 assert(batch < (long)INT_MAX + 1); 381 } 382 break; 383 case 'r': 384 if (!optarg) { 385 reset = 1; 386 } else { 387 reset = strtol(optarg, NULL, 10); 388 assert(reset > 0); 389 assert(reset < (long)INT_MAX + 1); 390 } 391 break; 392 default: 393 assert(0); 394 break; 395 } 396 } 397 398 done: 399 vdev_info_init(&dev, features); 400 vq_info_add(&dev, 256); 401 run_test(&dev, &dev.vqs[0], delayed, batch, reset, 0x100000); 402 return 0; 403 } 404