1 // SPDX-License-Identifier: GPL-2.0 2 #define _GNU_SOURCE 3 #include <getopt.h> 4 #include <limits.h> 5 #include <string.h> 6 #include <poll.h> 7 #include <sys/eventfd.h> 8 #include <stdlib.h> 9 #include <assert.h> 10 #include <unistd.h> 11 #include <sys/ioctl.h> 12 #include <sys/stat.h> 13 #include <sys/types.h> 14 #include <fcntl.h> 15 #include <stdbool.h> 16 #include <linux/virtio_types.h> 17 #include <linux/vhost.h> 18 #include <linux/virtio.h> 19 #include <linux/virtio_ring.h> 20 #include "../../drivers/vhost/test.h" 21 22 #define RANDOM_BATCH -1 23 24 /* Unused */ 25 void *__kmalloc_fake, *__kfree_ignore_start, *__kfree_ignore_end; 26 27 struct vq_info { 28 int kick; 29 int call; 30 int num; 31 int idx; 32 void *ring; 33 /* copy used for control */ 34 struct vring vring; 35 struct virtqueue *vq; 36 }; 37 38 struct vdev_info { 39 struct virtio_device vdev; 40 int control; 41 struct pollfd fds[1]; 42 struct vq_info vqs[1]; 43 int nvqs; 44 void *buf; 45 size_t buf_size; 46 struct vhost_memory *mem; 47 }; 48 49 static const struct vhost_vring_file no_backend = { .fd = -1 }, 50 backend = { .fd = 1 }; 51 static const struct vhost_vring_state null_state = {}; 52 53 bool vq_notify(struct virtqueue *vq) 54 { 55 struct vq_info *info = vq->priv; 56 unsigned long long v = 1; 57 int r; 58 r = write(info->kick, &v, sizeof v); 59 assert(r == sizeof v); 60 return true; 61 } 62 63 void vq_callback(struct virtqueue *vq) 64 { 65 } 66 67 68 void vhost_vq_setup(struct vdev_info *dev, struct vq_info *info) 69 { 70 struct vhost_vring_state state = { .index = info->idx }; 71 struct vhost_vring_file file = { .index = info->idx }; 72 unsigned long long features = dev->vdev.features; 73 struct vhost_vring_addr addr = { 74 .index = info->idx, 75 .desc_user_addr = (uint64_t)(unsigned long)info->vring.desc, 76 .avail_user_addr = (uint64_t)(unsigned long)info->vring.avail, 77 .used_user_addr = (uint64_t)(unsigned long)info->vring.used, 78 }; 79 int r; 80 r = ioctl(dev->control, VHOST_SET_FEATURES, &features); 81 assert(r >= 0); 82 state.num = info->vring.num; 83 r = ioctl(dev->control, VHOST_SET_VRING_NUM, &state); 84 assert(r >= 0); 85 state.num = 0; 86 r = ioctl(dev->control, VHOST_SET_VRING_BASE, &state); 87 assert(r >= 0); 88 r = ioctl(dev->control, VHOST_SET_VRING_ADDR, &addr); 89 assert(r >= 0); 90 file.fd = info->kick; 91 r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file); 92 assert(r >= 0); 93 file.fd = info->call; 94 r = ioctl(dev->control, VHOST_SET_VRING_CALL, &file); 95 assert(r >= 0); 96 } 97 98 static void vq_reset(struct vq_info *info, int num, struct virtio_device *vdev) 99 { 100 if (info->vq) 101 vring_del_virtqueue(info->vq); 102 103 memset(info->ring, 0, vring_size(num, 4096)); 104 vring_init(&info->vring, num, info->ring, 4096); 105 info->vq = __vring_new_virtqueue(info->idx, info->vring, vdev, true, 106 false, vq_notify, vq_callback, "test"); 107 assert(info->vq); 108 info->vq->priv = info; 109 } 110 111 static void vq_info_add(struct vdev_info *dev, int num) 112 { 113 struct vq_info *info = &dev->vqs[dev->nvqs]; 114 int r; 115 info->idx = dev->nvqs; 116 info->kick = eventfd(0, EFD_NONBLOCK); 117 info->call = eventfd(0, EFD_NONBLOCK); 118 r = posix_memalign(&info->ring, 4096, vring_size(num, 4096)); 119 assert(r >= 0); 120 vq_reset(info, num, &dev->vdev); 121 vhost_vq_setup(dev, info); 122 dev->fds[info->idx].fd = info->call; 123 dev->fds[info->idx].events = POLLIN; 124 dev->nvqs++; 125 } 126 127 static void vdev_info_init(struct vdev_info* dev, unsigned long long features) 128 { 129 int r; 130 memset(dev, 0, sizeof *dev); 131 dev->vdev.features = features; 132 INIT_LIST_HEAD(&dev->vdev.vqs); 133 dev->buf_size = 1024; 134 dev->buf = malloc(dev->buf_size); 135 assert(dev->buf); 136 dev->control = open("/dev/vhost-test", O_RDWR); 137 assert(dev->control >= 0); 138 r = ioctl(dev->control, VHOST_SET_OWNER, NULL); 139 assert(r >= 0); 140 dev->mem = malloc(offsetof(struct vhost_memory, regions) + 141 sizeof dev->mem->regions[0]); 142 assert(dev->mem); 143 memset(dev->mem, 0, offsetof(struct vhost_memory, regions) + 144 sizeof dev->mem->regions[0]); 145 dev->mem->nregions = 1; 146 dev->mem->regions[0].guest_phys_addr = (long)dev->buf; 147 dev->mem->regions[0].userspace_addr = (long)dev->buf; 148 dev->mem->regions[0].memory_size = dev->buf_size; 149 r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem); 150 assert(r >= 0); 151 } 152 153 /* TODO: this is pretty bad: we get a cache line bounce 154 * for the wait queue on poll and another one on read, 155 * plus the read which is there just to clear the 156 * current state. */ 157 static void wait_for_interrupt(struct vdev_info *dev) 158 { 159 int i; 160 unsigned long long val; 161 poll(dev->fds, dev->nvqs, -1); 162 for (i = 0; i < dev->nvqs; ++i) 163 if (dev->fds[i].revents & POLLIN) { 164 read(dev->fds[i].fd, &val, sizeof val); 165 } 166 } 167 168 static void run_test(struct vdev_info *dev, struct vq_info *vq, 169 bool delayed, int batch, int reset_n, int bufs) 170 { 171 struct scatterlist sl; 172 long started = 0, completed = 0, next_reset = reset_n; 173 long completed_before, started_before; 174 int r, test = 1; 175 unsigned len; 176 long long spurious = 0; 177 const bool random_batch = batch == RANDOM_BATCH; 178 179 r = ioctl(dev->control, VHOST_TEST_RUN, &test); 180 assert(r >= 0); 181 if (!reset_n) { 182 next_reset = INT_MAX; 183 } 184 185 for (;;) { 186 virtqueue_disable_cb(vq->vq); 187 completed_before = completed; 188 started_before = started; 189 do { 190 const bool reset = completed > next_reset; 191 if (random_batch) 192 batch = (random() % vq->vring.num) + 1; 193 194 while (started < bufs && 195 (started - completed) < batch) { 196 sg_init_one(&sl, dev->buf, dev->buf_size); 197 r = virtqueue_add_outbuf(vq->vq, &sl, 1, 198 dev->buf + started, 199 GFP_ATOMIC); 200 if (unlikely(r != 0)) { 201 if (r == -ENOSPC && 202 started > started_before) 203 r = 0; 204 else 205 r = -1; 206 break; 207 } 208 209 ++started; 210 211 if (unlikely(!virtqueue_kick(vq->vq))) { 212 r = -1; 213 break; 214 } 215 } 216 217 if (started >= bufs) 218 r = -1; 219 220 if (reset) { 221 r = ioctl(dev->control, VHOST_TEST_SET_BACKEND, 222 &no_backend); 223 assert(!r); 224 } 225 226 /* Flush out completed bufs if any */ 227 while (virtqueue_get_buf(vq->vq, &len)) { 228 ++completed; 229 r = 0; 230 } 231 232 if (reset) { 233 struct vhost_vring_state s = { .index = 0 }; 234 235 vq_reset(vq, vq->vring.num, &dev->vdev); 236 237 r = ioctl(dev->control, VHOST_GET_VRING_BASE, 238 &s); 239 assert(!r); 240 241 s.num = 0; 242 r = ioctl(dev->control, VHOST_SET_VRING_BASE, 243 &null_state); 244 assert(!r); 245 246 r = ioctl(dev->control, VHOST_TEST_SET_BACKEND, 247 &backend); 248 assert(!r); 249 250 started = completed; 251 while (completed > next_reset) 252 next_reset += completed; 253 } 254 } while (r == 0); 255 if (completed == completed_before && started == started_before) 256 ++spurious; 257 assert(completed <= bufs); 258 assert(started <= bufs); 259 if (completed == bufs) 260 break; 261 if (delayed) { 262 if (virtqueue_enable_cb_delayed(vq->vq)) 263 wait_for_interrupt(dev); 264 } else { 265 if (virtqueue_enable_cb(vq->vq)) 266 wait_for_interrupt(dev); 267 } 268 } 269 test = 0; 270 r = ioctl(dev->control, VHOST_TEST_RUN, &test); 271 assert(r >= 0); 272 fprintf(stderr, 273 "spurious wakeups: 0x%llx started=0x%lx completed=0x%lx\n", 274 spurious, started, completed); 275 } 276 277 const char optstring[] = "h"; 278 const struct option longopts[] = { 279 { 280 .name = "help", 281 .val = 'h', 282 }, 283 { 284 .name = "event-idx", 285 .val = 'E', 286 }, 287 { 288 .name = "no-event-idx", 289 .val = 'e', 290 }, 291 { 292 .name = "indirect", 293 .val = 'I', 294 }, 295 { 296 .name = "no-indirect", 297 .val = 'i', 298 }, 299 { 300 .name = "virtio-1", 301 .val = '1', 302 }, 303 { 304 .name = "no-virtio-1", 305 .val = '0', 306 }, 307 { 308 .name = "delayed-interrupt", 309 .val = 'D', 310 }, 311 { 312 .name = "no-delayed-interrupt", 313 .val = 'd', 314 }, 315 { 316 .name = "batch", 317 .val = 'b', 318 .has_arg = required_argument, 319 }, 320 { 321 .name = "reset", 322 .val = 'r', 323 .has_arg = optional_argument, 324 }, 325 { 326 } 327 }; 328 329 static void help(void) 330 { 331 fprintf(stderr, "Usage: virtio_test [--help]" 332 " [--no-indirect]" 333 " [--no-event-idx]" 334 " [--no-virtio-1]" 335 " [--delayed-interrupt]" 336 " [--batch=random/N]" 337 " [--reset=N]" 338 "\n"); 339 } 340 341 int main(int argc, char **argv) 342 { 343 struct vdev_info dev; 344 unsigned long long features = (1ULL << VIRTIO_RING_F_INDIRECT_DESC) | 345 (1ULL << VIRTIO_RING_F_EVENT_IDX) | (1ULL << VIRTIO_F_VERSION_1); 346 long batch = 1, reset = 0; 347 int o; 348 bool delayed = false; 349 350 for (;;) { 351 o = getopt_long(argc, argv, optstring, longopts, NULL); 352 switch (o) { 353 case -1: 354 goto done; 355 case '?': 356 help(); 357 exit(2); 358 case 'e': 359 features &= ~(1ULL << VIRTIO_RING_F_EVENT_IDX); 360 break; 361 case 'h': 362 help(); 363 goto done; 364 case 'i': 365 features &= ~(1ULL << VIRTIO_RING_F_INDIRECT_DESC); 366 break; 367 case '0': 368 features &= ~(1ULL << VIRTIO_F_VERSION_1); 369 break; 370 case 'D': 371 delayed = true; 372 break; 373 case 'b': 374 if (0 == strcmp(optarg, "random")) { 375 batch = RANDOM_BATCH; 376 } else { 377 batch = strtol(optarg, NULL, 10); 378 assert(batch > 0); 379 assert(batch < (long)INT_MAX + 1); 380 } 381 break; 382 case 'r': 383 if (!optarg) { 384 reset = 1; 385 } else { 386 reset = strtol(optarg, NULL, 10); 387 assert(reset > 0); 388 assert(reset < (long)INT_MAX + 1); 389 } 390 break; 391 default: 392 assert(0); 393 break; 394 } 395 } 396 397 done: 398 vdev_info_init(&dev, features); 399 vq_info_add(&dev, 256); 400 run_test(&dev, &dev.vqs[0], delayed, batch, reset, 0x100000); 401 return 0; 402 } 403