1 /* 2 * vhost-vdpa 3 * 4 * Copyright(c) 2017-2018 Intel Corporation. 5 * Copyright(c) 2020 Red Hat, Inc. 6 * 7 * This work is licensed under the terms of the GNU GPL, version 2 or later. 8 * See the COPYING file in the top-level directory. 9 * 10 */ 11 12 #include "qemu/osdep.h" 13 #include <linux/vhost.h> 14 #include <linux/vfio.h> 15 #include <sys/eventfd.h> 16 #include <sys/ioctl.h> 17 #include "hw/virtio/vhost.h" 18 #include "hw/virtio/vhost-backend.h" 19 #include "hw/virtio/virtio-net.h" 20 #include "hw/virtio/vhost-vdpa.h" 21 #include "qemu/main-loop.h" 22 #include "cpu.h" 23 24 static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section) 25 { 26 return (!memory_region_is_ram(section->mr) && 27 !memory_region_is_iommu(section->mr)) || 28 /* 29 * Sizing an enabled 64-bit BAR can cause spurious mappings to 30 * addresses in the upper part of the 64-bit address space. These 31 * are never accessed by the CPU and beyond the address width of 32 * some IOMMU hardware. TODO: VDPA should tell us the IOMMU width. 33 */ 34 section->offset_within_address_space & (1ULL << 63); 35 } 36 37 static int vhost_vdpa_dma_map(struct vhost_vdpa *v, hwaddr iova, hwaddr size, 38 void *vaddr, bool readonly) 39 { 40 struct vhost_msg_v2 msg = {}; 41 int fd = v->device_fd; 42 int ret = 0; 43 44 msg.type = v->msg_type; 45 msg.iotlb.iova = iova; 46 msg.iotlb.size = size; 47 msg.iotlb.uaddr = (uint64_t)(uintptr_t)vaddr; 48 msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW; 49 msg.iotlb.type = VHOST_IOTLB_UPDATE; 50 51 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 52 error_report("failed to write, fd=%d, errno=%d (%s)", 53 fd, errno, strerror(errno)); 54 return -EIO ; 55 } 56 57 return ret; 58 } 59 60 static int vhost_vdpa_dma_unmap(struct vhost_vdpa *v, hwaddr iova, 61 hwaddr size) 62 { 63 struct vhost_msg_v2 msg = {}; 64 int fd = v->device_fd; 65 int ret = 0; 66 67 msg.type = v->msg_type; 68 msg.iotlb.iova = iova; 69 msg.iotlb.size = size; 70 msg.iotlb.type = VHOST_IOTLB_INVALIDATE; 71 72 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 73 error_report("failed to write, fd=%d, errno=%d (%s)", 74 fd, errno, strerror(errno)); 75 return -EIO ; 76 } 77 78 return ret; 79 } 80 81 static void vhost_vdpa_listener_region_add(MemoryListener *listener, 82 MemoryRegionSection *section) 83 { 84 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 85 hwaddr iova; 86 Int128 llend, llsize; 87 void *vaddr; 88 int ret; 89 90 if (vhost_vdpa_listener_skipped_section(section)) { 91 return; 92 } 93 94 if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) != 95 (section->offset_within_region & ~TARGET_PAGE_MASK))) { 96 error_report("%s received unaligned region", __func__); 97 return; 98 } 99 100 iova = TARGET_PAGE_ALIGN(section->offset_within_address_space); 101 llend = int128_make64(section->offset_within_address_space); 102 llend = int128_add(llend, section->size); 103 llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK)); 104 105 if (int128_ge(int128_make64(iova), llend)) { 106 return; 107 } 108 109 memory_region_ref(section->mr); 110 111 /* Here we assume that memory_region_is_ram(section->mr)==true */ 112 113 vaddr = memory_region_get_ram_ptr(section->mr) + 114 section->offset_within_region + 115 (iova - section->offset_within_address_space); 116 117 llsize = int128_sub(llend, int128_make64(iova)); 118 119 ret = vhost_vdpa_dma_map(v, iova, int128_get64(llsize), 120 vaddr, section->readonly); 121 if (ret) { 122 error_report("vhost vdpa map fail!"); 123 if (memory_region_is_ram_device(section->mr)) { 124 /* Allow unexpected mappings not to be fatal for RAM devices */ 125 error_report("map ram fail!"); 126 return ; 127 } 128 goto fail; 129 } 130 131 return; 132 133 fail: 134 if (memory_region_is_ram_device(section->mr)) { 135 error_report("failed to vdpa_dma_map. pci p2p may not work"); 136 return; 137 138 } 139 /* 140 * On the initfn path, store the first error in the container so we 141 * can gracefully fail. Runtime, there's not much we can do other 142 * than throw a hardware error. 143 */ 144 error_report("vhost-vdpa: DMA mapping failed, unable to continue"); 145 return; 146 147 } 148 149 static void vhost_vdpa_listener_region_del(MemoryListener *listener, 150 MemoryRegionSection *section) 151 { 152 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 153 hwaddr iova; 154 Int128 llend, llsize; 155 int ret; 156 bool try_unmap = true; 157 158 if (vhost_vdpa_listener_skipped_section(section)) { 159 return; 160 } 161 162 if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) != 163 (section->offset_within_region & ~TARGET_PAGE_MASK))) { 164 error_report("%s received unaligned region", __func__); 165 return; 166 } 167 168 iova = TARGET_PAGE_ALIGN(section->offset_within_address_space); 169 llend = int128_make64(section->offset_within_address_space); 170 llend = int128_add(llend, section->size); 171 llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK)); 172 173 if (int128_ge(int128_make64(iova), llend)) { 174 return; 175 } 176 177 llsize = int128_sub(llend, int128_make64(iova)); 178 179 if (try_unmap) { 180 ret = vhost_vdpa_dma_unmap(v, iova, int128_get64(llsize)); 181 if (ret) { 182 error_report("vhost_vdpa dma unmap error!"); 183 } 184 } 185 186 memory_region_unref(section->mr); 187 } 188 /* 189 * IOTLB API is used by vhost-vpda which requires incremental updating 190 * of the mapping. So we can not use generic vhost memory listener which 191 * depends on the addnop(). 192 */ 193 static const MemoryListener vhost_vdpa_memory_listener = { 194 .region_add = vhost_vdpa_listener_region_add, 195 .region_del = vhost_vdpa_listener_region_del, 196 }; 197 198 static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request, 199 void *arg) 200 { 201 struct vhost_vdpa *v = dev->opaque; 202 int fd = v->device_fd; 203 204 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 205 206 return ioctl(fd, request, arg); 207 } 208 209 static void vhost_vdpa_add_status(struct vhost_dev *dev, uint8_t status) 210 { 211 uint8_t s; 212 213 if (vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s)) { 214 return; 215 } 216 217 s |= status; 218 219 vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &s); 220 } 221 222 static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque) 223 { 224 struct vhost_vdpa *v; 225 uint64_t features; 226 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 227 228 v = opaque; 229 dev->opaque = opaque ; 230 vhost_vdpa_call(dev, VHOST_GET_FEATURES, &features); 231 dev->backend_features = features; 232 v->listener = vhost_vdpa_memory_listener; 233 v->msg_type = VHOST_IOTLB_MSG_V2; 234 235 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | 236 VIRTIO_CONFIG_S_DRIVER); 237 238 return 0; 239 } 240 241 static int vhost_vdpa_cleanup(struct vhost_dev *dev) 242 { 243 struct vhost_vdpa *v; 244 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 245 v = dev->opaque; 246 memory_listener_unregister(&v->listener); 247 248 dev->opaque = NULL; 249 return 0; 250 } 251 252 static int vhost_vdpa_memslots_limit(struct vhost_dev *dev) 253 { 254 return INT_MAX; 255 } 256 257 static int vhost_vdpa_set_mem_table(struct vhost_dev *dev, 258 struct vhost_memory *mem) 259 { 260 261 if (mem->padding) { 262 return -1; 263 } 264 265 return 0; 266 } 267 268 static int vhost_vdpa_set_features(struct vhost_dev *dev, 269 uint64_t features) 270 { 271 int ret; 272 ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features); 273 uint8_t status = 0; 274 if (ret) { 275 return ret; 276 } 277 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK); 278 vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &status); 279 280 return !(status & VIRTIO_CONFIG_S_FEATURES_OK); 281 } 282 283 int vhost_vdpa_get_device_id(struct vhost_dev *dev, 284 uint32_t *device_id) 285 { 286 return vhost_vdpa_call(dev, VHOST_VDPA_GET_DEVICE_ID, device_id); 287 } 288 289 static int vhost_vdpa_reset_device(struct vhost_dev *dev) 290 { 291 uint8_t status = 0; 292 293 return vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status); 294 } 295 296 static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx) 297 { 298 assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs); 299 300 return idx - dev->vq_index; 301 } 302 303 static int vhost_vdpa_set_vring_ready(struct vhost_dev *dev) 304 { 305 int i; 306 for (i = 0; i < dev->nvqs; ++i) { 307 struct vhost_vring_state state = { 308 .index = dev->vq_index + i, 309 .num = 1, 310 }; 311 vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state); 312 } 313 return 0; 314 } 315 316 static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data, 317 uint32_t offset, uint32_t size, 318 uint32_t flags) 319 { 320 struct vhost_vdpa_config *config; 321 int ret; 322 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf); 323 config = g_malloc(size + config_size); 324 if (config == NULL) { 325 return -1; 326 } 327 config->off = offset; 328 config->len = size; 329 memcpy(config->buf, data, size); 330 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG, config); 331 g_free(config); 332 return ret; 333 } 334 335 static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config, 336 uint32_t config_len) 337 { 338 struct vhost_vdpa_config *v_config; 339 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf); 340 int ret; 341 342 v_config = g_malloc(config_len + config_size); 343 if (v_config == NULL) { 344 return -1; 345 } 346 v_config->len = config_len; 347 v_config->off = 0; 348 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_CONFIG, v_config); 349 memcpy(config, v_config->buf, config_len); 350 g_free(v_config); 351 return ret; 352 } 353 354 static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started) 355 { 356 struct vhost_vdpa *v = dev->opaque; 357 if (started) { 358 uint8_t status = 0; 359 memory_listener_register(&v->listener, &address_space_memory); 360 vhost_vdpa_set_vring_ready(dev); 361 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK); 362 vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &status); 363 364 return !(status & VIRTIO_CONFIG_S_DRIVER_OK); 365 } else { 366 vhost_vdpa_reset_device(dev); 367 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | 368 VIRTIO_CONFIG_S_DRIVER); 369 memory_listener_unregister(&v->listener); 370 371 return 0; 372 } 373 } 374 375 static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base, 376 struct vhost_log *log) 377 { 378 return vhost_vdpa_call(dev, VHOST_SET_LOG_BASE, &base); 379 } 380 381 static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev, 382 struct vhost_vring_addr *addr) 383 { 384 return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr); 385 } 386 387 static int vhost_vdpa_set_vring_num(struct vhost_dev *dev, 388 struct vhost_vring_state *ring) 389 { 390 return vhost_vdpa_call(dev, VHOST_SET_VRING_NUM, ring); 391 } 392 393 static int vhost_vdpa_set_vring_base(struct vhost_dev *dev, 394 struct vhost_vring_state *ring) 395 { 396 return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring); 397 } 398 399 static int vhost_vdpa_get_vring_base(struct vhost_dev *dev, 400 struct vhost_vring_state *ring) 401 { 402 return vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring); 403 } 404 405 static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev, 406 struct vhost_vring_file *file) 407 { 408 return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file); 409 } 410 411 static int vhost_vdpa_set_vring_call(struct vhost_dev *dev, 412 struct vhost_vring_file *file) 413 { 414 return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file); 415 } 416 417 static int vhost_vdpa_get_features(struct vhost_dev *dev, 418 uint64_t *features) 419 { 420 return vhost_vdpa_call(dev, VHOST_GET_FEATURES, features); 421 } 422 423 static int vhost_vdpa_set_owner(struct vhost_dev *dev) 424 { 425 return vhost_vdpa_call(dev, VHOST_SET_OWNER, NULL); 426 } 427 428 static int vhost_vdpa_vq_get_addr(struct vhost_dev *dev, 429 struct vhost_vring_addr *addr, struct vhost_virtqueue *vq) 430 { 431 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 432 addr->desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys; 433 addr->avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys; 434 addr->used_user_addr = (uint64_t)(unsigned long)vq->used_phys; 435 return 0; 436 } 437 438 static bool vhost_vdpa_force_iommu(struct vhost_dev *dev) 439 { 440 return true; 441 } 442 443 const VhostOps vdpa_ops = { 444 .backend_type = VHOST_BACKEND_TYPE_VDPA, 445 .vhost_backend_init = vhost_vdpa_init, 446 .vhost_backend_cleanup = vhost_vdpa_cleanup, 447 .vhost_set_log_base = vhost_vdpa_set_log_base, 448 .vhost_set_vring_addr = vhost_vdpa_set_vring_addr, 449 .vhost_set_vring_num = vhost_vdpa_set_vring_num, 450 .vhost_set_vring_base = vhost_vdpa_set_vring_base, 451 .vhost_get_vring_base = vhost_vdpa_get_vring_base, 452 .vhost_set_vring_kick = vhost_vdpa_set_vring_kick, 453 .vhost_set_vring_call = vhost_vdpa_set_vring_call, 454 .vhost_get_features = vhost_vdpa_get_features, 455 .vhost_set_owner = vhost_vdpa_set_owner, 456 .vhost_set_vring_endian = NULL, 457 .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit, 458 .vhost_set_mem_table = vhost_vdpa_set_mem_table, 459 .vhost_set_features = vhost_vdpa_set_features, 460 .vhost_reset_device = vhost_vdpa_reset_device, 461 .vhost_get_vq_index = vhost_vdpa_get_vq_index, 462 .vhost_get_config = vhost_vdpa_get_config, 463 .vhost_set_config = vhost_vdpa_set_config, 464 .vhost_requires_shm_log = NULL, 465 .vhost_migration_done = NULL, 466 .vhost_backend_can_merge = NULL, 467 .vhost_net_set_mtu = NULL, 468 .vhost_set_iotlb_callback = NULL, 469 .vhost_send_device_iotlb_msg = NULL, 470 .vhost_dev_start = vhost_vdpa_dev_start, 471 .vhost_get_device_id = vhost_vdpa_get_device_id, 472 .vhost_vq_get_addr = vhost_vdpa_vq_get_addr, 473 .vhost_force_iommu = vhost_vdpa_force_iommu, 474 }; 475