1 /* 2 * Virtio Support 3 * 4 * Copyright IBM, Corp. 2007 5 * 6 * Authors: 7 * Anthony Liguori <aliguori@us.ibm.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 */ 13 14 #include "qemu/osdep.h" 15 #include "qapi/error.h" 16 #include "qapi/qmp/qdict.h" 17 #include "qapi/qapi-commands-virtio.h" 18 #include "qapi/qapi-commands-qom.h" 19 #include "qapi/qapi-visit-virtio.h" 20 #include "qapi/qmp/qjson.h" 21 #include "cpu.h" 22 #include "trace.h" 23 #include "qemu/error-report.h" 24 #include "qemu/log.h" 25 #include "qemu/main-loop.h" 26 #include "qemu/module.h" 27 #include "qom/object_interfaces.h" 28 #include "hw/virtio/virtio.h" 29 #include "migration/qemu-file-types.h" 30 #include "qemu/atomic.h" 31 #include "hw/virtio/virtio-bus.h" 32 #include "hw/qdev-properties.h" 33 #include "hw/virtio/virtio-access.h" 34 #include "sysemu/dma.h" 35 #include "sysemu/runstate.h" 36 #include "standard-headers/linux/virtio_ids.h" 37 #include "standard-headers/linux/vhost_types.h" 38 #include "standard-headers/linux/virtio_blk.h" 39 #include "standard-headers/linux/virtio_console.h" 40 #include "standard-headers/linux/virtio_gpu.h" 41 #include "standard-headers/linux/virtio_net.h" 42 #include "standard-headers/linux/virtio_scsi.h" 43 #include "standard-headers/linux/virtio_i2c.h" 44 #include "standard-headers/linux/virtio_balloon.h" 45 #include "standard-headers/linux/virtio_iommu.h" 46 #include "standard-headers/linux/virtio_mem.h" 47 #include "standard-headers/linux/virtio_vsock.h" 48 #include CONFIG_DEVICES 49 50 /* QAPI list of realized VirtIODevices */ 51 static QTAILQ_HEAD(, VirtIODevice) virtio_list; 52 53 /* 54 * Maximum size of virtio device config space 55 */ 56 #define VHOST_USER_MAX_CONFIG_SIZE 256 57 58 #define FEATURE_ENTRY(name, desc) (qmp_virtio_feature_map_t) \ 59 { .virtio_bit = name, .feature_desc = desc } 60 61 enum VhostUserProtocolFeature { 62 VHOST_USER_PROTOCOL_F_MQ = 0, 63 VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1, 64 VHOST_USER_PROTOCOL_F_RARP = 2, 65 VHOST_USER_PROTOCOL_F_REPLY_ACK = 3, 66 VHOST_USER_PROTOCOL_F_NET_MTU = 4, 67 VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5, 68 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6, 69 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7, 70 VHOST_USER_PROTOCOL_F_PAGEFAULT = 8, 71 VHOST_USER_PROTOCOL_F_CONFIG = 9, 72 VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10, 73 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11, 74 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD = 12, 75 VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13, 76 VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS = 14, 77 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS = 15, 78 VHOST_USER_PROTOCOL_F_MAX 79 }; 80 81 /* Virtio transport features mapping */ 82 static qmp_virtio_feature_map_t virtio_transport_map[] = { 83 /* Virtio device transport features */ 84 #ifndef VIRTIO_CONFIG_NO_LEGACY 85 FEATURE_ENTRY(VIRTIO_F_NOTIFY_ON_EMPTY, \ 86 "VIRTIO_F_NOTIFY_ON_EMPTY: Notify when device runs out of avail. " 87 "descs. on VQ"), 88 FEATURE_ENTRY(VIRTIO_F_ANY_LAYOUT, \ 89 "VIRTIO_F_ANY_LAYOUT: Device accepts arbitrary desc. layouts"), 90 #endif /* !VIRTIO_CONFIG_NO_LEGACY */ 91 FEATURE_ENTRY(VIRTIO_F_VERSION_1, \ 92 "VIRTIO_F_VERSION_1: Device compliant for v1 spec (legacy)"), 93 FEATURE_ENTRY(VIRTIO_F_IOMMU_PLATFORM, \ 94 "VIRTIO_F_IOMMU_PLATFORM: Device can be used on IOMMU platform"), 95 FEATURE_ENTRY(VIRTIO_F_RING_PACKED, \ 96 "VIRTIO_F_RING_PACKED: Device supports packed VQ layout"), 97 FEATURE_ENTRY(VIRTIO_F_IN_ORDER, \ 98 "VIRTIO_F_IN_ORDER: Device uses buffers in same order as made " 99 "available by driver"), 100 FEATURE_ENTRY(VIRTIO_F_ORDER_PLATFORM, \ 101 "VIRTIO_F_ORDER_PLATFORM: Memory accesses ordered by platform"), 102 FEATURE_ENTRY(VIRTIO_F_SR_IOV, \ 103 "VIRTIO_F_SR_IOV: Device supports single root I/O virtualization"), 104 /* Virtio ring transport features */ 105 FEATURE_ENTRY(VIRTIO_RING_F_INDIRECT_DESC, \ 106 "VIRTIO_RING_F_INDIRECT_DESC: Indirect descriptors supported"), 107 FEATURE_ENTRY(VIRTIO_RING_F_EVENT_IDX, \ 108 "VIRTIO_RING_F_EVENT_IDX: Used & avail. event fields enabled"), 109 { -1, "" } 110 }; 111 112 /* Vhost-user protocol features mapping */ 113 static qmp_virtio_feature_map_t vhost_user_protocol_map[] = { 114 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_MQ, \ 115 "VHOST_USER_PROTOCOL_F_MQ: Multiqueue protocol supported"), 116 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_LOG_SHMFD, \ 117 "VHOST_USER_PROTOCOL_F_LOG_SHMFD: Shared log memory fd supported"), 118 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_RARP, \ 119 "VHOST_USER_PROTOCOL_F_RARP: Vhost-user back-end RARP broadcasting " 120 "supported"), 121 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_REPLY_ACK, \ 122 "VHOST_USER_PROTOCOL_F_REPLY_ACK: Requested operation status ack. " 123 "supported"), 124 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_NET_MTU, \ 125 "VHOST_USER_PROTOCOL_F_NET_MTU: Expose host MTU to guest supported"), 126 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_SLAVE_REQ, \ 127 "VHOST_USER_PROTOCOL_F_SLAVE_REQ: Socket fd for back-end initiated " 128 "requests supported"), 129 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_CROSS_ENDIAN, \ 130 "VHOST_USER_PROTOCOL_F_CROSS_ENDIAN: Endianness of VQs for legacy " 131 "devices supported"), 132 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_CRYPTO_SESSION, \ 133 "VHOST_USER_PROTOCOL_F_CRYPTO_SESSION: Session creation for crypto " 134 "operations supported"), 135 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_PAGEFAULT, \ 136 "VHOST_USER_PROTOCOL_F_PAGEFAULT: Request servicing on userfaultfd " 137 "for accessed pages supported"), 138 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_CONFIG, \ 139 "VHOST_USER_PROTOCOL_F_CONFIG: Vhost-user messaging for virtio " 140 "device configuration space supported"), 141 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD, \ 142 "VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD: Slave fd communication " 143 "channel supported"), 144 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_HOST_NOTIFIER, \ 145 "VHOST_USER_PROTOCOL_F_HOST_NOTIFIER: Host notifiers for specified " 146 "VQs supported"), 147 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD, \ 148 "VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD: Shared inflight I/O buffers " 149 "supported"), 150 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_RESET_DEVICE, \ 151 "VHOST_USER_PROTOCOL_F_RESET_DEVICE: Disabling all rings and " 152 "resetting internal device state supported"), 153 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS, \ 154 "VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS: In-band messaging " 155 "supported"), 156 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS, \ 157 "VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS: Configuration for " 158 "memory slots supported"), 159 { -1, "" } 160 }; 161 162 /* virtio device configuration statuses */ 163 static qmp_virtio_feature_map_t virtio_config_status_map[] = { 164 FEATURE_ENTRY(VIRTIO_CONFIG_S_DRIVER_OK, \ 165 "VIRTIO_CONFIG_S_DRIVER_OK: Driver setup and ready"), 166 FEATURE_ENTRY(VIRTIO_CONFIG_S_FEATURES_OK, \ 167 "VIRTIO_CONFIG_S_FEATURES_OK: Feature negotiation complete"), 168 FEATURE_ENTRY(VIRTIO_CONFIG_S_DRIVER, \ 169 "VIRTIO_CONFIG_S_DRIVER: Guest OS compatible with device"), 170 FEATURE_ENTRY(VIRTIO_CONFIG_S_NEEDS_RESET, \ 171 "VIRTIO_CONFIG_S_NEEDS_RESET: Irrecoverable error, device needs " 172 "reset"), 173 FEATURE_ENTRY(VIRTIO_CONFIG_S_FAILED, \ 174 "VIRTIO_CONFIG_S_FAILED: Error in guest, device failed"), 175 FEATURE_ENTRY(VIRTIO_CONFIG_S_ACKNOWLEDGE, \ 176 "VIRTIO_CONFIG_S_ACKNOWLEDGE: Valid virtio device found"), 177 { -1, "" } 178 }; 179 180 /* virtio-blk features mapping */ 181 qmp_virtio_feature_map_t virtio_blk_feature_map[] = { 182 FEATURE_ENTRY(VIRTIO_BLK_F_SIZE_MAX, \ 183 "VIRTIO_BLK_F_SIZE_MAX: Max segment size is size_max"), 184 FEATURE_ENTRY(VIRTIO_BLK_F_SEG_MAX, \ 185 "VIRTIO_BLK_F_SEG_MAX: Max segments in a request is seg_max"), 186 FEATURE_ENTRY(VIRTIO_BLK_F_GEOMETRY, \ 187 "VIRTIO_BLK_F_GEOMETRY: Legacy geometry available"), 188 FEATURE_ENTRY(VIRTIO_BLK_F_RO, \ 189 "VIRTIO_BLK_F_RO: Device is read-only"), 190 FEATURE_ENTRY(VIRTIO_BLK_F_BLK_SIZE, \ 191 "VIRTIO_BLK_F_BLK_SIZE: Block size of disk available"), 192 FEATURE_ENTRY(VIRTIO_BLK_F_TOPOLOGY, \ 193 "VIRTIO_BLK_F_TOPOLOGY: Topology information available"), 194 FEATURE_ENTRY(VIRTIO_BLK_F_MQ, \ 195 "VIRTIO_BLK_F_MQ: Multiqueue supported"), 196 FEATURE_ENTRY(VIRTIO_BLK_F_DISCARD, \ 197 "VIRTIO_BLK_F_DISCARD: Discard command supported"), 198 FEATURE_ENTRY(VIRTIO_BLK_F_WRITE_ZEROES, \ 199 "VIRTIO_BLK_F_WRITE_ZEROES: Write zeroes command supported"), 200 #ifndef VIRTIO_BLK_NO_LEGACY 201 FEATURE_ENTRY(VIRTIO_BLK_F_BARRIER, \ 202 "VIRTIO_BLK_F_BARRIER: Request barriers supported"), 203 FEATURE_ENTRY(VIRTIO_BLK_F_SCSI, \ 204 "VIRTIO_BLK_F_SCSI: SCSI packet commands supported"), 205 FEATURE_ENTRY(VIRTIO_BLK_F_FLUSH, \ 206 "VIRTIO_BLK_F_FLUSH: Flush command supported"), 207 FEATURE_ENTRY(VIRTIO_BLK_F_CONFIG_WCE, \ 208 "VIRTIO_BLK_F_CONFIG_WCE: Cache writeback and writethrough modes " 209 "supported"), 210 #endif /* !VIRTIO_BLK_NO_LEGACY */ 211 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 212 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 213 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 214 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 215 "negotiation supported"), 216 { -1, "" } 217 }; 218 219 /* virtio-serial features mapping */ 220 qmp_virtio_feature_map_t virtio_serial_feature_map[] = { 221 FEATURE_ENTRY(VIRTIO_CONSOLE_F_SIZE, \ 222 "VIRTIO_CONSOLE_F_SIZE: Host providing console size"), 223 FEATURE_ENTRY(VIRTIO_CONSOLE_F_MULTIPORT, \ 224 "VIRTIO_CONSOLE_F_MULTIPORT: Multiple ports for device supported"), 225 FEATURE_ENTRY(VIRTIO_CONSOLE_F_EMERG_WRITE, \ 226 "VIRTIO_CONSOLE_F_EMERG_WRITE: Emergency write supported"), 227 { -1, "" } 228 }; 229 230 /* virtio-gpu features mapping */ 231 qmp_virtio_feature_map_t virtio_gpu_feature_map[] = { 232 FEATURE_ENTRY(VIRTIO_GPU_F_VIRGL, \ 233 "VIRTIO_GPU_F_VIRGL: Virgl 3D mode supported"), 234 FEATURE_ENTRY(VIRTIO_GPU_F_EDID, \ 235 "VIRTIO_GPU_F_EDID: EDID metadata supported"), 236 FEATURE_ENTRY(VIRTIO_GPU_F_RESOURCE_UUID, \ 237 "VIRTIO_GPU_F_RESOURCE_UUID: Resource UUID assigning supported"), 238 FEATURE_ENTRY(VIRTIO_GPU_F_RESOURCE_BLOB, \ 239 "VIRTIO_GPU_F_RESOURCE_BLOB: Size-based blob resources supported"), 240 FEATURE_ENTRY(VIRTIO_GPU_F_CONTEXT_INIT, \ 241 "VIRTIO_GPU_F_CONTEXT_INIT: Context types and synchronization " 242 "timelines supported"), 243 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 244 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 245 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 246 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 247 "negotiation supported"), 248 { -1, "" } 249 }; 250 251 /* virtio-input features mapping */ 252 qmp_virtio_feature_map_t virtio_input_feature_map[] = { 253 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 254 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 255 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 256 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 257 "negotiation supported"), 258 { -1, "" } 259 }; 260 261 /* virtio-net features mapping */ 262 qmp_virtio_feature_map_t virtio_net_feature_map[] = { 263 FEATURE_ENTRY(VIRTIO_NET_F_CSUM, \ 264 "VIRTIO_NET_F_CSUM: Device handling packets with partial checksum " 265 "supported"), 266 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_CSUM, \ 267 "VIRTIO_NET_F_GUEST_CSUM: Driver handling packets with partial " 268 "checksum supported"), 269 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \ 270 "VIRTIO_NET_F_CTRL_GUEST_OFFLOADS: Control channel offloading " 271 "reconfig. supported"), 272 FEATURE_ENTRY(VIRTIO_NET_F_MTU, \ 273 "VIRTIO_NET_F_MTU: Device max MTU reporting supported"), 274 FEATURE_ENTRY(VIRTIO_NET_F_MAC, \ 275 "VIRTIO_NET_F_MAC: Device has given MAC address"), 276 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_TSO4, \ 277 "VIRTIO_NET_F_GUEST_TSO4: Driver can receive TSOv4"), 278 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_TSO6, \ 279 "VIRTIO_NET_F_GUEST_TSO6: Driver can receive TSOv6"), 280 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_ECN, \ 281 "VIRTIO_NET_F_GUEST_ECN: Driver can receive TSO with ECN"), 282 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_UFO, \ 283 "VIRTIO_NET_F_GUEST_UFO: Driver can receive UFO"), 284 FEATURE_ENTRY(VIRTIO_NET_F_HOST_TSO4, \ 285 "VIRTIO_NET_F_HOST_TSO4: Device can receive TSOv4"), 286 FEATURE_ENTRY(VIRTIO_NET_F_HOST_TSO6, \ 287 "VIRTIO_NET_F_HOST_TSO6: Device can receive TSOv6"), 288 FEATURE_ENTRY(VIRTIO_NET_F_HOST_ECN, \ 289 "VIRTIO_NET_F_HOST_ECN: Device can receive TSO with ECN"), 290 FEATURE_ENTRY(VIRTIO_NET_F_HOST_UFO, \ 291 "VIRTIO_NET_F_HOST_UFO: Device can receive UFO"), 292 FEATURE_ENTRY(VIRTIO_NET_F_MRG_RXBUF, \ 293 "VIRTIO_NET_F_MRG_RXBUF: Driver can merge receive buffers"), 294 FEATURE_ENTRY(VIRTIO_NET_F_STATUS, \ 295 "VIRTIO_NET_F_STATUS: Configuration status field available"), 296 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_VQ, \ 297 "VIRTIO_NET_F_CTRL_VQ: Control channel available"), 298 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_RX, \ 299 "VIRTIO_NET_F_CTRL_RX: Control channel RX mode supported"), 300 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_VLAN, \ 301 "VIRTIO_NET_F_CTRL_VLAN: Control channel VLAN filtering supported"), 302 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_RX_EXTRA, \ 303 "VIRTIO_NET_F_CTRL_RX_EXTRA: Extra RX mode control supported"), 304 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_ANNOUNCE, \ 305 "VIRTIO_NET_F_GUEST_ANNOUNCE: Driver sending gratuitous packets " 306 "supported"), 307 FEATURE_ENTRY(VIRTIO_NET_F_MQ, \ 308 "VIRTIO_NET_F_MQ: Multiqueue with automatic receive steering " 309 "supported"), 310 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_MAC_ADDR, \ 311 "VIRTIO_NET_F_CTRL_MAC_ADDR: MAC address set through control " 312 "channel"), 313 FEATURE_ENTRY(VIRTIO_NET_F_HASH_REPORT, \ 314 "VIRTIO_NET_F_HASH_REPORT: Hash reporting supported"), 315 FEATURE_ENTRY(VIRTIO_NET_F_RSS, \ 316 "VIRTIO_NET_F_RSS: RSS RX steering supported"), 317 FEATURE_ENTRY(VIRTIO_NET_F_RSC_EXT, \ 318 "VIRTIO_NET_F_RSC_EXT: Extended coalescing info supported"), 319 FEATURE_ENTRY(VIRTIO_NET_F_STANDBY, \ 320 "VIRTIO_NET_F_STANDBY: Device acting as standby for primary " 321 "device with same MAC addr. supported"), 322 FEATURE_ENTRY(VIRTIO_NET_F_SPEED_DUPLEX, \ 323 "VIRTIO_NET_F_SPEED_DUPLEX: Device set linkspeed and duplex"), 324 #ifndef VIRTIO_NET_NO_LEGACY 325 FEATURE_ENTRY(VIRTIO_NET_F_GSO, \ 326 "VIRTIO_NET_F_GSO: Handling GSO-type packets supported"), 327 #endif /* !VIRTIO_NET_NO_LEGACY */ 328 FEATURE_ENTRY(VHOST_NET_F_VIRTIO_NET_HDR, \ 329 "VHOST_NET_F_VIRTIO_NET_HDR: Virtio-net headers for RX and TX " 330 "packets supported"), 331 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 332 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 333 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 334 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 335 "negotiation supported"), 336 { -1, "" } 337 }; 338 339 /* virtio-scsi features mapping */ 340 qmp_virtio_feature_map_t virtio_scsi_feature_map[] = { 341 FEATURE_ENTRY(VIRTIO_SCSI_F_INOUT, \ 342 "VIRTIO_SCSI_F_INOUT: Requests including read and writable data " 343 "buffers suppoted"), 344 FEATURE_ENTRY(VIRTIO_SCSI_F_HOTPLUG, \ 345 "VIRTIO_SCSI_F_HOTPLUG: Reporting and handling hot-plug events " 346 "supported"), 347 FEATURE_ENTRY(VIRTIO_SCSI_F_CHANGE, \ 348 "VIRTIO_SCSI_F_CHANGE: Reporting and handling LUN changes " 349 "supported"), 350 FEATURE_ENTRY(VIRTIO_SCSI_F_T10_PI, \ 351 "VIRTIO_SCSI_F_T10_PI: T10 info included in request header"), 352 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 353 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 354 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 355 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 356 "negotiation supported"), 357 { -1, "" } 358 }; 359 360 /* virtio/vhost-user-fs features mapping */ 361 qmp_virtio_feature_map_t virtio_fs_feature_map[] = { 362 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 363 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 364 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 365 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 366 "negotiation supported"), 367 { -1, "" } 368 }; 369 370 /* virtio/vhost-user-i2c features mapping */ 371 qmp_virtio_feature_map_t virtio_i2c_feature_map[] = { 372 FEATURE_ENTRY(VIRTIO_I2C_F_ZERO_LENGTH_REQUEST, \ 373 "VIRTIO_I2C_F_ZERO_LEGNTH_REQUEST: Zero length requests supported"), 374 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 375 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 376 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 377 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 378 "negotiation supported"), 379 { -1, "" } 380 }; 381 382 /* virtio/vhost-vsock features mapping */ 383 qmp_virtio_feature_map_t virtio_vsock_feature_map[] = { 384 FEATURE_ENTRY(VIRTIO_VSOCK_F_SEQPACKET, \ 385 "VIRTIO_VSOCK_F_SEQPACKET: SOCK_SEQPACKET supported"), 386 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 387 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 388 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 389 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 390 "negotiation supported"), 391 { -1, "" } 392 }; 393 394 /* virtio-balloon features mapping */ 395 qmp_virtio_feature_map_t virtio_balloon_feature_map[] = { 396 FEATURE_ENTRY(VIRTIO_BALLOON_F_MUST_TELL_HOST, \ 397 "VIRTIO_BALLOON_F_MUST_TELL_HOST: Tell host before reclaiming " 398 "pages"), 399 FEATURE_ENTRY(VIRTIO_BALLOON_F_STATS_VQ, \ 400 "VIRTIO_BALLOON_F_STATS_VQ: Guest memory stats VQ available"), 401 FEATURE_ENTRY(VIRTIO_BALLOON_F_DEFLATE_ON_OOM, \ 402 "VIRTIO_BALLOON_F_DEFLATE_ON_OOM: Deflate balloon when guest OOM"), 403 FEATURE_ENTRY(VIRTIO_BALLOON_F_FREE_PAGE_HINT, \ 404 "VIRTIO_BALLOON_F_FREE_PAGE_HINT: VQ reporting free pages enabled"), 405 FEATURE_ENTRY(VIRTIO_BALLOON_F_PAGE_POISON, \ 406 "VIRTIO_BALLOON_F_PAGE_POISON: Guest page poisoning enabled"), 407 FEATURE_ENTRY(VIRTIO_BALLOON_F_REPORTING, \ 408 "VIRTIO_BALLOON_F_REPORTING: Page reporting VQ enabled"), 409 { -1, "" } 410 }; 411 412 /* virtio-crypto features mapping */ 413 qmp_virtio_feature_map_t virtio_crypto_feature_map[] = { 414 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 415 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 416 { -1, "" } 417 }; 418 419 /* virtio-iommu features mapping */ 420 qmp_virtio_feature_map_t virtio_iommu_feature_map[] = { 421 FEATURE_ENTRY(VIRTIO_IOMMU_F_INPUT_RANGE, \ 422 "VIRTIO_IOMMU_F_INPUT_RANGE: Range of available virtual addrs. " 423 "available"), 424 FEATURE_ENTRY(VIRTIO_IOMMU_F_DOMAIN_RANGE, \ 425 "VIRTIO_IOMMU_F_DOMAIN_RANGE: Number of supported domains " 426 "available"), 427 FEATURE_ENTRY(VIRTIO_IOMMU_F_MAP_UNMAP, \ 428 "VIRTIO_IOMMU_F_MAP_UNMAP: Map and unmap requests available"), 429 FEATURE_ENTRY(VIRTIO_IOMMU_F_BYPASS, \ 430 "VIRTIO_IOMMU_F_BYPASS: Endpoints not attached to domains are in " 431 "bypass mode"), 432 FEATURE_ENTRY(VIRTIO_IOMMU_F_PROBE, \ 433 "VIRTIO_IOMMU_F_PROBE: Probe requests available"), 434 FEATURE_ENTRY(VIRTIO_IOMMU_F_MMIO, \ 435 "VIRTIO_IOMMU_F_MMIO: VIRTIO_IOMMU_MAP_F_MMIO flag available"), 436 FEATURE_ENTRY(VIRTIO_IOMMU_F_BYPASS_CONFIG, \ 437 "VIRTIO_IOMMU_F_BYPASS_CONFIG: Bypass field of IOMMU config " 438 "available"), 439 { -1, "" } 440 }; 441 442 /* virtio-mem features mapping */ 443 qmp_virtio_feature_map_t virtio_mem_feature_map[] = { 444 #ifndef CONFIG_ACPI 445 FEATURE_ENTRY(VIRTIO_MEM_F_ACPI_PXM, \ 446 "VIRTIO_MEM_F_ACPI_PXM: node_id is an ACPI PXM and is valid"), 447 #endif /* !CONFIG_ACPI */ 448 FEATURE_ENTRY(VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE, \ 449 "VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE: Unplugged memory cannot be " 450 "accessed"), 451 { -1, "" } 452 }; 453 454 /* virtio-rng features mapping */ 455 qmp_virtio_feature_map_t virtio_rng_feature_map[] = { 456 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 457 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 458 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 459 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 460 "negotiation supported"), 461 { -1, "" } 462 }; 463 464 /* 465 * The alignment to use between consumer and producer parts of vring. 466 * x86 pagesize again. This is the default, used by transports like PCI 467 * which don't provide a means for the guest to tell the host the alignment. 468 */ 469 #define VIRTIO_PCI_VRING_ALIGN 4096 470 471 typedef struct VRingDesc 472 { 473 uint64_t addr; 474 uint32_t len; 475 uint16_t flags; 476 uint16_t next; 477 } VRingDesc; 478 479 typedef struct VRingPackedDesc { 480 uint64_t addr; 481 uint32_t len; 482 uint16_t id; 483 uint16_t flags; 484 } VRingPackedDesc; 485 486 typedef struct VRingAvail 487 { 488 uint16_t flags; 489 uint16_t idx; 490 uint16_t ring[]; 491 } VRingAvail; 492 493 typedef struct VRingUsedElem 494 { 495 uint32_t id; 496 uint32_t len; 497 } VRingUsedElem; 498 499 typedef struct VRingUsed 500 { 501 uint16_t flags; 502 uint16_t idx; 503 VRingUsedElem ring[]; 504 } VRingUsed; 505 506 typedef struct VRingMemoryRegionCaches { 507 struct rcu_head rcu; 508 MemoryRegionCache desc; 509 MemoryRegionCache avail; 510 MemoryRegionCache used; 511 } VRingMemoryRegionCaches; 512 513 typedef struct VRing 514 { 515 unsigned int num; 516 unsigned int num_default; 517 unsigned int align; 518 hwaddr desc; 519 hwaddr avail; 520 hwaddr used; 521 VRingMemoryRegionCaches *caches; 522 } VRing; 523 524 typedef struct VRingPackedDescEvent { 525 uint16_t off_wrap; 526 uint16_t flags; 527 } VRingPackedDescEvent ; 528 529 struct VirtQueue 530 { 531 VRing vring; 532 VirtQueueElement *used_elems; 533 534 /* Next head to pop */ 535 uint16_t last_avail_idx; 536 bool last_avail_wrap_counter; 537 538 /* Last avail_idx read from VQ. */ 539 uint16_t shadow_avail_idx; 540 bool shadow_avail_wrap_counter; 541 542 uint16_t used_idx; 543 bool used_wrap_counter; 544 545 /* Last used index value we have signalled on */ 546 uint16_t signalled_used; 547 548 /* Last used index value we have signalled on */ 549 bool signalled_used_valid; 550 551 /* Notification enabled? */ 552 bool notification; 553 554 uint16_t queue_index; 555 556 unsigned int inuse; 557 558 uint16_t vector; 559 VirtIOHandleOutput handle_output; 560 VirtIODevice *vdev; 561 EventNotifier guest_notifier; 562 EventNotifier host_notifier; 563 bool host_notifier_enabled; 564 QLIST_ENTRY(VirtQueue) node; 565 }; 566 567 const char *virtio_device_names[] = { 568 [VIRTIO_ID_NET] = "virtio-net", 569 [VIRTIO_ID_BLOCK] = "virtio-blk", 570 [VIRTIO_ID_CONSOLE] = "virtio-serial", 571 [VIRTIO_ID_RNG] = "virtio-rng", 572 [VIRTIO_ID_BALLOON] = "virtio-balloon", 573 [VIRTIO_ID_IOMEM] = "virtio-iomem", 574 [VIRTIO_ID_RPMSG] = "virtio-rpmsg", 575 [VIRTIO_ID_SCSI] = "virtio-scsi", 576 [VIRTIO_ID_9P] = "virtio-9p", 577 [VIRTIO_ID_MAC80211_WLAN] = "virtio-mac-wlan", 578 [VIRTIO_ID_RPROC_SERIAL] = "virtio-rproc-serial", 579 [VIRTIO_ID_CAIF] = "virtio-caif", 580 [VIRTIO_ID_MEMORY_BALLOON] = "virtio-mem-balloon", 581 [VIRTIO_ID_GPU] = "virtio-gpu", 582 [VIRTIO_ID_CLOCK] = "virtio-clk", 583 [VIRTIO_ID_INPUT] = "virtio-input", 584 [VIRTIO_ID_VSOCK] = "vhost-vsock", 585 [VIRTIO_ID_CRYPTO] = "virtio-crypto", 586 [VIRTIO_ID_SIGNAL_DIST] = "virtio-signal", 587 [VIRTIO_ID_PSTORE] = "virtio-pstore", 588 [VIRTIO_ID_IOMMU] = "virtio-iommu", 589 [VIRTIO_ID_MEM] = "virtio-mem", 590 [VIRTIO_ID_SOUND] = "virtio-sound", 591 [VIRTIO_ID_FS] = "virtio-user-fs", 592 [VIRTIO_ID_PMEM] = "virtio-pmem", 593 [VIRTIO_ID_RPMB] = "virtio-rpmb", 594 [VIRTIO_ID_MAC80211_HWSIM] = "virtio-mac-hwsim", 595 [VIRTIO_ID_VIDEO_ENCODER] = "virtio-vid-encoder", 596 [VIRTIO_ID_VIDEO_DECODER] = "virtio-vid-decoder", 597 [VIRTIO_ID_SCMI] = "virtio-scmi", 598 [VIRTIO_ID_NITRO_SEC_MOD] = "virtio-nitro-sec-mod", 599 [VIRTIO_ID_I2C_ADAPTER] = "vhost-user-i2c", 600 [VIRTIO_ID_WATCHDOG] = "virtio-watchdog", 601 [VIRTIO_ID_CAN] = "virtio-can", 602 [VIRTIO_ID_DMABUF] = "virtio-dmabuf", 603 [VIRTIO_ID_PARAM_SERV] = "virtio-param-serv", 604 [VIRTIO_ID_AUDIO_POLICY] = "virtio-audio-pol", 605 [VIRTIO_ID_BT] = "virtio-bluetooth", 606 [VIRTIO_ID_GPIO] = "virtio-gpio" 607 }; 608 609 static const char *virtio_id_to_name(uint16_t device_id) 610 { 611 assert(device_id < G_N_ELEMENTS(virtio_device_names)); 612 const char *name = virtio_device_names[device_id]; 613 assert(name != NULL); 614 return name; 615 } 616 617 /* Called within call_rcu(). */ 618 static void virtio_free_region_cache(VRingMemoryRegionCaches *caches) 619 { 620 assert(caches != NULL); 621 address_space_cache_destroy(&caches->desc); 622 address_space_cache_destroy(&caches->avail); 623 address_space_cache_destroy(&caches->used); 624 g_free(caches); 625 } 626 627 static void virtio_virtqueue_reset_region_cache(struct VirtQueue *vq) 628 { 629 VRingMemoryRegionCaches *caches; 630 631 caches = qatomic_read(&vq->vring.caches); 632 qatomic_rcu_set(&vq->vring.caches, NULL); 633 if (caches) { 634 call_rcu(caches, virtio_free_region_cache, rcu); 635 } 636 } 637 638 static void virtio_init_region_cache(VirtIODevice *vdev, int n) 639 { 640 VirtQueue *vq = &vdev->vq[n]; 641 VRingMemoryRegionCaches *old = vq->vring.caches; 642 VRingMemoryRegionCaches *new = NULL; 643 hwaddr addr, size; 644 int64_t len; 645 bool packed; 646 647 648 addr = vq->vring.desc; 649 if (!addr) { 650 goto out_no_cache; 651 } 652 new = g_new0(VRingMemoryRegionCaches, 1); 653 size = virtio_queue_get_desc_size(vdev, n); 654 packed = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ? 655 true : false; 656 len = address_space_cache_init(&new->desc, vdev->dma_as, 657 addr, size, packed); 658 if (len < size) { 659 virtio_error(vdev, "Cannot map desc"); 660 goto err_desc; 661 } 662 663 size = virtio_queue_get_used_size(vdev, n); 664 len = address_space_cache_init(&new->used, vdev->dma_as, 665 vq->vring.used, size, true); 666 if (len < size) { 667 virtio_error(vdev, "Cannot map used"); 668 goto err_used; 669 } 670 671 size = virtio_queue_get_avail_size(vdev, n); 672 len = address_space_cache_init(&new->avail, vdev->dma_as, 673 vq->vring.avail, size, false); 674 if (len < size) { 675 virtio_error(vdev, "Cannot map avail"); 676 goto err_avail; 677 } 678 679 qatomic_rcu_set(&vq->vring.caches, new); 680 if (old) { 681 call_rcu(old, virtio_free_region_cache, rcu); 682 } 683 return; 684 685 err_avail: 686 address_space_cache_destroy(&new->avail); 687 err_used: 688 address_space_cache_destroy(&new->used); 689 err_desc: 690 address_space_cache_destroy(&new->desc); 691 out_no_cache: 692 g_free(new); 693 virtio_virtqueue_reset_region_cache(vq); 694 } 695 696 /* virt queue functions */ 697 void virtio_queue_update_rings(VirtIODevice *vdev, int n) 698 { 699 VRing *vring = &vdev->vq[n].vring; 700 701 if (!vring->num || !vring->desc || !vring->align) { 702 /* not yet setup -> nothing to do */ 703 return; 704 } 705 vring->avail = vring->desc + vring->num * sizeof(VRingDesc); 706 vring->used = vring_align(vring->avail + 707 offsetof(VRingAvail, ring[vring->num]), 708 vring->align); 709 virtio_init_region_cache(vdev, n); 710 } 711 712 /* Called within rcu_read_lock(). */ 713 static void vring_split_desc_read(VirtIODevice *vdev, VRingDesc *desc, 714 MemoryRegionCache *cache, int i) 715 { 716 address_space_read_cached(cache, i * sizeof(VRingDesc), 717 desc, sizeof(VRingDesc)); 718 virtio_tswap64s(vdev, &desc->addr); 719 virtio_tswap32s(vdev, &desc->len); 720 virtio_tswap16s(vdev, &desc->flags); 721 virtio_tswap16s(vdev, &desc->next); 722 } 723 724 static void vring_packed_event_read(VirtIODevice *vdev, 725 MemoryRegionCache *cache, 726 VRingPackedDescEvent *e) 727 { 728 hwaddr off_off = offsetof(VRingPackedDescEvent, off_wrap); 729 hwaddr off_flags = offsetof(VRingPackedDescEvent, flags); 730 731 e->flags = virtio_lduw_phys_cached(vdev, cache, off_flags); 732 /* Make sure flags is seen before off_wrap */ 733 smp_rmb(); 734 e->off_wrap = virtio_lduw_phys_cached(vdev, cache, off_off); 735 virtio_tswap16s(vdev, &e->flags); 736 } 737 738 static void vring_packed_off_wrap_write(VirtIODevice *vdev, 739 MemoryRegionCache *cache, 740 uint16_t off_wrap) 741 { 742 hwaddr off = offsetof(VRingPackedDescEvent, off_wrap); 743 744 virtio_stw_phys_cached(vdev, cache, off, off_wrap); 745 address_space_cache_invalidate(cache, off, sizeof(off_wrap)); 746 } 747 748 static void vring_packed_flags_write(VirtIODevice *vdev, 749 MemoryRegionCache *cache, uint16_t flags) 750 { 751 hwaddr off = offsetof(VRingPackedDescEvent, flags); 752 753 virtio_stw_phys_cached(vdev, cache, off, flags); 754 address_space_cache_invalidate(cache, off, sizeof(flags)); 755 } 756 757 /* Called within rcu_read_lock(). */ 758 static VRingMemoryRegionCaches *vring_get_region_caches(struct VirtQueue *vq) 759 { 760 return qatomic_rcu_read(&vq->vring.caches); 761 } 762 763 /* Called within rcu_read_lock(). */ 764 static inline uint16_t vring_avail_flags(VirtQueue *vq) 765 { 766 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 767 hwaddr pa = offsetof(VRingAvail, flags); 768 769 if (!caches) { 770 return 0; 771 } 772 773 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); 774 } 775 776 /* Called within rcu_read_lock(). */ 777 static inline uint16_t vring_avail_idx(VirtQueue *vq) 778 { 779 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 780 hwaddr pa = offsetof(VRingAvail, idx); 781 782 if (!caches) { 783 return 0; 784 } 785 786 vq->shadow_avail_idx = virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); 787 return vq->shadow_avail_idx; 788 } 789 790 /* Called within rcu_read_lock(). */ 791 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i) 792 { 793 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 794 hwaddr pa = offsetof(VRingAvail, ring[i]); 795 796 if (!caches) { 797 return 0; 798 } 799 800 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); 801 } 802 803 /* Called within rcu_read_lock(). */ 804 static inline uint16_t vring_get_used_event(VirtQueue *vq) 805 { 806 return vring_avail_ring(vq, vq->vring.num); 807 } 808 809 /* Called within rcu_read_lock(). */ 810 static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem, 811 int i) 812 { 813 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 814 hwaddr pa = offsetof(VRingUsed, ring[i]); 815 816 if (!caches) { 817 return; 818 } 819 820 virtio_tswap32s(vq->vdev, &uelem->id); 821 virtio_tswap32s(vq->vdev, &uelem->len); 822 address_space_write_cached(&caches->used, pa, uelem, sizeof(VRingUsedElem)); 823 address_space_cache_invalidate(&caches->used, pa, sizeof(VRingUsedElem)); 824 } 825 826 /* Called within rcu_read_lock(). */ 827 static uint16_t vring_used_idx(VirtQueue *vq) 828 { 829 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 830 hwaddr pa = offsetof(VRingUsed, idx); 831 832 if (!caches) { 833 return 0; 834 } 835 836 return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); 837 } 838 839 /* Called within rcu_read_lock(). */ 840 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val) 841 { 842 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 843 hwaddr pa = offsetof(VRingUsed, idx); 844 845 if (caches) { 846 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val); 847 address_space_cache_invalidate(&caches->used, pa, sizeof(val)); 848 } 849 850 vq->used_idx = val; 851 } 852 853 /* Called within rcu_read_lock(). */ 854 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask) 855 { 856 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 857 VirtIODevice *vdev = vq->vdev; 858 hwaddr pa = offsetof(VRingUsed, flags); 859 uint16_t flags; 860 861 if (!caches) { 862 return; 863 } 864 865 flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); 866 virtio_stw_phys_cached(vdev, &caches->used, pa, flags | mask); 867 address_space_cache_invalidate(&caches->used, pa, sizeof(flags)); 868 } 869 870 /* Called within rcu_read_lock(). */ 871 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask) 872 { 873 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 874 VirtIODevice *vdev = vq->vdev; 875 hwaddr pa = offsetof(VRingUsed, flags); 876 uint16_t flags; 877 878 if (!caches) { 879 return; 880 } 881 882 flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); 883 virtio_stw_phys_cached(vdev, &caches->used, pa, flags & ~mask); 884 address_space_cache_invalidate(&caches->used, pa, sizeof(flags)); 885 } 886 887 /* Called within rcu_read_lock(). */ 888 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val) 889 { 890 VRingMemoryRegionCaches *caches; 891 hwaddr pa; 892 if (!vq->notification) { 893 return; 894 } 895 896 caches = vring_get_region_caches(vq); 897 if (!caches) { 898 return; 899 } 900 901 pa = offsetof(VRingUsed, ring[vq->vring.num]); 902 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val); 903 address_space_cache_invalidate(&caches->used, pa, sizeof(val)); 904 } 905 906 static void virtio_queue_split_set_notification(VirtQueue *vq, int enable) 907 { 908 RCU_READ_LOCK_GUARD(); 909 910 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { 911 vring_set_avail_event(vq, vring_avail_idx(vq)); 912 } else if (enable) { 913 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY); 914 } else { 915 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY); 916 } 917 if (enable) { 918 /* Expose avail event/used flags before caller checks the avail idx. */ 919 smp_mb(); 920 } 921 } 922 923 static void virtio_queue_packed_set_notification(VirtQueue *vq, int enable) 924 { 925 uint16_t off_wrap; 926 VRingPackedDescEvent e; 927 VRingMemoryRegionCaches *caches; 928 929 RCU_READ_LOCK_GUARD(); 930 caches = vring_get_region_caches(vq); 931 if (!caches) { 932 return; 933 } 934 935 vring_packed_event_read(vq->vdev, &caches->used, &e); 936 937 if (!enable) { 938 e.flags = VRING_PACKED_EVENT_FLAG_DISABLE; 939 } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { 940 off_wrap = vq->shadow_avail_idx | vq->shadow_avail_wrap_counter << 15; 941 vring_packed_off_wrap_write(vq->vdev, &caches->used, off_wrap); 942 /* Make sure off_wrap is wrote before flags */ 943 smp_wmb(); 944 e.flags = VRING_PACKED_EVENT_FLAG_DESC; 945 } else { 946 e.flags = VRING_PACKED_EVENT_FLAG_ENABLE; 947 } 948 949 vring_packed_flags_write(vq->vdev, &caches->used, e.flags); 950 if (enable) { 951 /* Expose avail event/used flags before caller checks the avail idx. */ 952 smp_mb(); 953 } 954 } 955 956 bool virtio_queue_get_notification(VirtQueue *vq) 957 { 958 return vq->notification; 959 } 960 961 void virtio_queue_set_notification(VirtQueue *vq, int enable) 962 { 963 vq->notification = enable; 964 965 if (!vq->vring.desc) { 966 return; 967 } 968 969 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 970 virtio_queue_packed_set_notification(vq, enable); 971 } else { 972 virtio_queue_split_set_notification(vq, enable); 973 } 974 } 975 976 int virtio_queue_ready(VirtQueue *vq) 977 { 978 return vq->vring.avail != 0; 979 } 980 981 static void vring_packed_desc_read_flags(VirtIODevice *vdev, 982 uint16_t *flags, 983 MemoryRegionCache *cache, 984 int i) 985 { 986 hwaddr off = i * sizeof(VRingPackedDesc) + offsetof(VRingPackedDesc, flags); 987 988 *flags = virtio_lduw_phys_cached(vdev, cache, off); 989 } 990 991 static void vring_packed_desc_read(VirtIODevice *vdev, 992 VRingPackedDesc *desc, 993 MemoryRegionCache *cache, 994 int i, bool strict_order) 995 { 996 hwaddr off = i * sizeof(VRingPackedDesc); 997 998 vring_packed_desc_read_flags(vdev, &desc->flags, cache, i); 999 1000 if (strict_order) { 1001 /* Make sure flags is read before the rest fields. */ 1002 smp_rmb(); 1003 } 1004 1005 address_space_read_cached(cache, off + offsetof(VRingPackedDesc, addr), 1006 &desc->addr, sizeof(desc->addr)); 1007 address_space_read_cached(cache, off + offsetof(VRingPackedDesc, id), 1008 &desc->id, sizeof(desc->id)); 1009 address_space_read_cached(cache, off + offsetof(VRingPackedDesc, len), 1010 &desc->len, sizeof(desc->len)); 1011 virtio_tswap64s(vdev, &desc->addr); 1012 virtio_tswap16s(vdev, &desc->id); 1013 virtio_tswap32s(vdev, &desc->len); 1014 } 1015 1016 static void vring_packed_desc_write_data(VirtIODevice *vdev, 1017 VRingPackedDesc *desc, 1018 MemoryRegionCache *cache, 1019 int i) 1020 { 1021 hwaddr off_id = i * sizeof(VRingPackedDesc) + 1022 offsetof(VRingPackedDesc, id); 1023 hwaddr off_len = i * sizeof(VRingPackedDesc) + 1024 offsetof(VRingPackedDesc, len); 1025 1026 virtio_tswap32s(vdev, &desc->len); 1027 virtio_tswap16s(vdev, &desc->id); 1028 address_space_write_cached(cache, off_id, &desc->id, sizeof(desc->id)); 1029 address_space_cache_invalidate(cache, off_id, sizeof(desc->id)); 1030 address_space_write_cached(cache, off_len, &desc->len, sizeof(desc->len)); 1031 address_space_cache_invalidate(cache, off_len, sizeof(desc->len)); 1032 } 1033 1034 static void vring_packed_desc_write_flags(VirtIODevice *vdev, 1035 VRingPackedDesc *desc, 1036 MemoryRegionCache *cache, 1037 int i) 1038 { 1039 hwaddr off = i * sizeof(VRingPackedDesc) + offsetof(VRingPackedDesc, flags); 1040 1041 virtio_stw_phys_cached(vdev, cache, off, desc->flags); 1042 address_space_cache_invalidate(cache, off, sizeof(desc->flags)); 1043 } 1044 1045 static void vring_packed_desc_write(VirtIODevice *vdev, 1046 VRingPackedDesc *desc, 1047 MemoryRegionCache *cache, 1048 int i, bool strict_order) 1049 { 1050 vring_packed_desc_write_data(vdev, desc, cache, i); 1051 if (strict_order) { 1052 /* Make sure data is wrote before flags. */ 1053 smp_wmb(); 1054 } 1055 vring_packed_desc_write_flags(vdev, desc, cache, i); 1056 } 1057 1058 static inline bool is_desc_avail(uint16_t flags, bool wrap_counter) 1059 { 1060 bool avail, used; 1061 1062 avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL)); 1063 used = !!(flags & (1 << VRING_PACKED_DESC_F_USED)); 1064 return (avail != used) && (avail == wrap_counter); 1065 } 1066 1067 /* Fetch avail_idx from VQ memory only when we really need to know if 1068 * guest has added some buffers. 1069 * Called within rcu_read_lock(). */ 1070 static int virtio_queue_empty_rcu(VirtQueue *vq) 1071 { 1072 if (virtio_device_disabled(vq->vdev)) { 1073 return 1; 1074 } 1075 1076 if (unlikely(!vq->vring.avail)) { 1077 return 1; 1078 } 1079 1080 if (vq->shadow_avail_idx != vq->last_avail_idx) { 1081 return 0; 1082 } 1083 1084 return vring_avail_idx(vq) == vq->last_avail_idx; 1085 } 1086 1087 static int virtio_queue_split_empty(VirtQueue *vq) 1088 { 1089 bool empty; 1090 1091 if (virtio_device_disabled(vq->vdev)) { 1092 return 1; 1093 } 1094 1095 if (unlikely(!vq->vring.avail)) { 1096 return 1; 1097 } 1098 1099 if (vq->shadow_avail_idx != vq->last_avail_idx) { 1100 return 0; 1101 } 1102 1103 RCU_READ_LOCK_GUARD(); 1104 empty = vring_avail_idx(vq) == vq->last_avail_idx; 1105 return empty; 1106 } 1107 1108 /* Called within rcu_read_lock(). */ 1109 static int virtio_queue_packed_empty_rcu(VirtQueue *vq) 1110 { 1111 struct VRingPackedDesc desc; 1112 VRingMemoryRegionCaches *cache; 1113 1114 if (unlikely(!vq->vring.desc)) { 1115 return 1; 1116 } 1117 1118 cache = vring_get_region_caches(vq); 1119 if (!cache) { 1120 return 1; 1121 } 1122 1123 vring_packed_desc_read_flags(vq->vdev, &desc.flags, &cache->desc, 1124 vq->last_avail_idx); 1125 1126 return !is_desc_avail(desc.flags, vq->last_avail_wrap_counter); 1127 } 1128 1129 static int virtio_queue_packed_empty(VirtQueue *vq) 1130 { 1131 RCU_READ_LOCK_GUARD(); 1132 return virtio_queue_packed_empty_rcu(vq); 1133 } 1134 1135 int virtio_queue_empty(VirtQueue *vq) 1136 { 1137 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1138 return virtio_queue_packed_empty(vq); 1139 } else { 1140 return virtio_queue_split_empty(vq); 1141 } 1142 } 1143 1144 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem, 1145 unsigned int len) 1146 { 1147 AddressSpace *dma_as = vq->vdev->dma_as; 1148 unsigned int offset; 1149 int i; 1150 1151 offset = 0; 1152 for (i = 0; i < elem->in_num; i++) { 1153 size_t size = MIN(len - offset, elem->in_sg[i].iov_len); 1154 1155 dma_memory_unmap(dma_as, elem->in_sg[i].iov_base, 1156 elem->in_sg[i].iov_len, 1157 DMA_DIRECTION_FROM_DEVICE, size); 1158 1159 offset += size; 1160 } 1161 1162 for (i = 0; i < elem->out_num; i++) 1163 dma_memory_unmap(dma_as, elem->out_sg[i].iov_base, 1164 elem->out_sg[i].iov_len, 1165 DMA_DIRECTION_TO_DEVICE, 1166 elem->out_sg[i].iov_len); 1167 } 1168 1169 /* virtqueue_detach_element: 1170 * @vq: The #VirtQueue 1171 * @elem: The #VirtQueueElement 1172 * @len: number of bytes written 1173 * 1174 * Detach the element from the virtqueue. This function is suitable for device 1175 * reset or other situations where a #VirtQueueElement is simply freed and will 1176 * not be pushed or discarded. 1177 */ 1178 void virtqueue_detach_element(VirtQueue *vq, const VirtQueueElement *elem, 1179 unsigned int len) 1180 { 1181 vq->inuse -= elem->ndescs; 1182 virtqueue_unmap_sg(vq, elem, len); 1183 } 1184 1185 static void virtqueue_split_rewind(VirtQueue *vq, unsigned int num) 1186 { 1187 vq->last_avail_idx -= num; 1188 } 1189 1190 static void virtqueue_packed_rewind(VirtQueue *vq, unsigned int num) 1191 { 1192 if (vq->last_avail_idx < num) { 1193 vq->last_avail_idx = vq->vring.num + vq->last_avail_idx - num; 1194 vq->last_avail_wrap_counter ^= 1; 1195 } else { 1196 vq->last_avail_idx -= num; 1197 } 1198 } 1199 1200 /* virtqueue_unpop: 1201 * @vq: The #VirtQueue 1202 * @elem: The #VirtQueueElement 1203 * @len: number of bytes written 1204 * 1205 * Pretend the most recent element wasn't popped from the virtqueue. The next 1206 * call to virtqueue_pop() will refetch the element. 1207 */ 1208 void virtqueue_unpop(VirtQueue *vq, const VirtQueueElement *elem, 1209 unsigned int len) 1210 { 1211 1212 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1213 virtqueue_packed_rewind(vq, 1); 1214 } else { 1215 virtqueue_split_rewind(vq, 1); 1216 } 1217 1218 virtqueue_detach_element(vq, elem, len); 1219 } 1220 1221 /* virtqueue_rewind: 1222 * @vq: The #VirtQueue 1223 * @num: Number of elements to push back 1224 * 1225 * Pretend that elements weren't popped from the virtqueue. The next 1226 * virtqueue_pop() will refetch the oldest element. 1227 * 1228 * Use virtqueue_unpop() instead if you have a VirtQueueElement. 1229 * 1230 * Returns: true on success, false if @num is greater than the number of in use 1231 * elements. 1232 */ 1233 bool virtqueue_rewind(VirtQueue *vq, unsigned int num) 1234 { 1235 if (num > vq->inuse) { 1236 return false; 1237 } 1238 1239 vq->inuse -= num; 1240 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1241 virtqueue_packed_rewind(vq, num); 1242 } else { 1243 virtqueue_split_rewind(vq, num); 1244 } 1245 return true; 1246 } 1247 1248 static void virtqueue_split_fill(VirtQueue *vq, const VirtQueueElement *elem, 1249 unsigned int len, unsigned int idx) 1250 { 1251 VRingUsedElem uelem; 1252 1253 if (unlikely(!vq->vring.used)) { 1254 return; 1255 } 1256 1257 idx = (idx + vq->used_idx) % vq->vring.num; 1258 1259 uelem.id = elem->index; 1260 uelem.len = len; 1261 vring_used_write(vq, &uelem, idx); 1262 } 1263 1264 static void virtqueue_packed_fill(VirtQueue *vq, const VirtQueueElement *elem, 1265 unsigned int len, unsigned int idx) 1266 { 1267 vq->used_elems[idx].index = elem->index; 1268 vq->used_elems[idx].len = len; 1269 vq->used_elems[idx].ndescs = elem->ndescs; 1270 } 1271 1272 static void virtqueue_packed_fill_desc(VirtQueue *vq, 1273 const VirtQueueElement *elem, 1274 unsigned int idx, 1275 bool strict_order) 1276 { 1277 uint16_t head; 1278 VRingMemoryRegionCaches *caches; 1279 VRingPackedDesc desc = { 1280 .id = elem->index, 1281 .len = elem->len, 1282 }; 1283 bool wrap_counter = vq->used_wrap_counter; 1284 1285 if (unlikely(!vq->vring.desc)) { 1286 return; 1287 } 1288 1289 head = vq->used_idx + idx; 1290 if (head >= vq->vring.num) { 1291 head -= vq->vring.num; 1292 wrap_counter ^= 1; 1293 } 1294 if (wrap_counter) { 1295 desc.flags |= (1 << VRING_PACKED_DESC_F_AVAIL); 1296 desc.flags |= (1 << VRING_PACKED_DESC_F_USED); 1297 } else { 1298 desc.flags &= ~(1 << VRING_PACKED_DESC_F_AVAIL); 1299 desc.flags &= ~(1 << VRING_PACKED_DESC_F_USED); 1300 } 1301 1302 caches = vring_get_region_caches(vq); 1303 if (!caches) { 1304 return; 1305 } 1306 1307 vring_packed_desc_write(vq->vdev, &desc, &caches->desc, head, strict_order); 1308 } 1309 1310 /* Called within rcu_read_lock(). */ 1311 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem, 1312 unsigned int len, unsigned int idx) 1313 { 1314 trace_virtqueue_fill(vq, elem, len, idx); 1315 1316 virtqueue_unmap_sg(vq, elem, len); 1317 1318 if (virtio_device_disabled(vq->vdev)) { 1319 return; 1320 } 1321 1322 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1323 virtqueue_packed_fill(vq, elem, len, idx); 1324 } else { 1325 virtqueue_split_fill(vq, elem, len, idx); 1326 } 1327 } 1328 1329 /* Called within rcu_read_lock(). */ 1330 static void virtqueue_split_flush(VirtQueue *vq, unsigned int count) 1331 { 1332 uint16_t old, new; 1333 1334 if (unlikely(!vq->vring.used)) { 1335 return; 1336 } 1337 1338 /* Make sure buffer is written before we update index. */ 1339 smp_wmb(); 1340 trace_virtqueue_flush(vq, count); 1341 old = vq->used_idx; 1342 new = old + count; 1343 vring_used_idx_set(vq, new); 1344 vq->inuse -= count; 1345 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old))) 1346 vq->signalled_used_valid = false; 1347 } 1348 1349 static void virtqueue_packed_flush(VirtQueue *vq, unsigned int count) 1350 { 1351 unsigned int i, ndescs = 0; 1352 1353 if (unlikely(!vq->vring.desc)) { 1354 return; 1355 } 1356 1357 for (i = 1; i < count; i++) { 1358 virtqueue_packed_fill_desc(vq, &vq->used_elems[i], i, false); 1359 ndescs += vq->used_elems[i].ndescs; 1360 } 1361 virtqueue_packed_fill_desc(vq, &vq->used_elems[0], 0, true); 1362 ndescs += vq->used_elems[0].ndescs; 1363 1364 vq->inuse -= ndescs; 1365 vq->used_idx += ndescs; 1366 if (vq->used_idx >= vq->vring.num) { 1367 vq->used_idx -= vq->vring.num; 1368 vq->used_wrap_counter ^= 1; 1369 vq->signalled_used_valid = false; 1370 } 1371 } 1372 1373 void virtqueue_flush(VirtQueue *vq, unsigned int count) 1374 { 1375 if (virtio_device_disabled(vq->vdev)) { 1376 vq->inuse -= count; 1377 return; 1378 } 1379 1380 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1381 virtqueue_packed_flush(vq, count); 1382 } else { 1383 virtqueue_split_flush(vq, count); 1384 } 1385 } 1386 1387 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem, 1388 unsigned int len) 1389 { 1390 RCU_READ_LOCK_GUARD(); 1391 virtqueue_fill(vq, elem, len, 0); 1392 virtqueue_flush(vq, 1); 1393 } 1394 1395 /* Called within rcu_read_lock(). */ 1396 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx) 1397 { 1398 uint16_t num_heads = vring_avail_idx(vq) - idx; 1399 1400 /* Check it isn't doing very strange things with descriptor numbers. */ 1401 if (num_heads > vq->vring.num) { 1402 virtio_error(vq->vdev, "Guest moved used index from %u to %u", 1403 idx, vq->shadow_avail_idx); 1404 return -EINVAL; 1405 } 1406 /* On success, callers read a descriptor at vq->last_avail_idx. 1407 * Make sure descriptor read does not bypass avail index read. */ 1408 if (num_heads) { 1409 smp_rmb(); 1410 } 1411 1412 return num_heads; 1413 } 1414 1415 /* Called within rcu_read_lock(). */ 1416 static bool virtqueue_get_head(VirtQueue *vq, unsigned int idx, 1417 unsigned int *head) 1418 { 1419 /* Grab the next descriptor number they're advertising, and increment 1420 * the index we've seen. */ 1421 *head = vring_avail_ring(vq, idx % vq->vring.num); 1422 1423 /* If their number is silly, that's a fatal mistake. */ 1424 if (*head >= vq->vring.num) { 1425 virtio_error(vq->vdev, "Guest says index %u is available", *head); 1426 return false; 1427 } 1428 1429 return true; 1430 } 1431 1432 enum { 1433 VIRTQUEUE_READ_DESC_ERROR = -1, 1434 VIRTQUEUE_READ_DESC_DONE = 0, /* end of chain */ 1435 VIRTQUEUE_READ_DESC_MORE = 1, /* more buffers in chain */ 1436 }; 1437 1438 static int virtqueue_split_read_next_desc(VirtIODevice *vdev, VRingDesc *desc, 1439 MemoryRegionCache *desc_cache, 1440 unsigned int max, unsigned int *next) 1441 { 1442 /* If this descriptor says it doesn't chain, we're done. */ 1443 if (!(desc->flags & VRING_DESC_F_NEXT)) { 1444 return VIRTQUEUE_READ_DESC_DONE; 1445 } 1446 1447 /* Check they're not leading us off end of descriptors. */ 1448 *next = desc->next; 1449 /* Make sure compiler knows to grab that: we don't want it changing! */ 1450 smp_wmb(); 1451 1452 if (*next >= max) { 1453 virtio_error(vdev, "Desc next is %u", *next); 1454 return VIRTQUEUE_READ_DESC_ERROR; 1455 } 1456 1457 vring_split_desc_read(vdev, desc, desc_cache, *next); 1458 return VIRTQUEUE_READ_DESC_MORE; 1459 } 1460 1461 /* Called within rcu_read_lock(). */ 1462 static void virtqueue_split_get_avail_bytes(VirtQueue *vq, 1463 unsigned int *in_bytes, unsigned int *out_bytes, 1464 unsigned max_in_bytes, unsigned max_out_bytes, 1465 VRingMemoryRegionCaches *caches) 1466 { 1467 VirtIODevice *vdev = vq->vdev; 1468 unsigned int max, idx; 1469 unsigned int total_bufs, in_total, out_total; 1470 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 1471 int64_t len = 0; 1472 int rc; 1473 1474 idx = vq->last_avail_idx; 1475 total_bufs = in_total = out_total = 0; 1476 1477 max = vq->vring.num; 1478 1479 while ((rc = virtqueue_num_heads(vq, idx)) > 0) { 1480 MemoryRegionCache *desc_cache = &caches->desc; 1481 unsigned int num_bufs; 1482 VRingDesc desc; 1483 unsigned int i; 1484 1485 num_bufs = total_bufs; 1486 1487 if (!virtqueue_get_head(vq, idx++, &i)) { 1488 goto err; 1489 } 1490 1491 vring_split_desc_read(vdev, &desc, desc_cache, i); 1492 1493 if (desc.flags & VRING_DESC_F_INDIRECT) { 1494 if (!desc.len || (desc.len % sizeof(VRingDesc))) { 1495 virtio_error(vdev, "Invalid size for indirect buffer table"); 1496 goto err; 1497 } 1498 1499 /* If we've got too many, that implies a descriptor loop. */ 1500 if (num_bufs >= max) { 1501 virtio_error(vdev, "Looped descriptor"); 1502 goto err; 1503 } 1504 1505 /* loop over the indirect descriptor table */ 1506 len = address_space_cache_init(&indirect_desc_cache, 1507 vdev->dma_as, 1508 desc.addr, desc.len, false); 1509 desc_cache = &indirect_desc_cache; 1510 if (len < desc.len) { 1511 virtio_error(vdev, "Cannot map indirect buffer"); 1512 goto err; 1513 } 1514 1515 max = desc.len / sizeof(VRingDesc); 1516 num_bufs = i = 0; 1517 vring_split_desc_read(vdev, &desc, desc_cache, i); 1518 } 1519 1520 do { 1521 /* If we've got too many, that implies a descriptor loop. */ 1522 if (++num_bufs > max) { 1523 virtio_error(vdev, "Looped descriptor"); 1524 goto err; 1525 } 1526 1527 if (desc.flags & VRING_DESC_F_WRITE) { 1528 in_total += desc.len; 1529 } else { 1530 out_total += desc.len; 1531 } 1532 if (in_total >= max_in_bytes && out_total >= max_out_bytes) { 1533 goto done; 1534 } 1535 1536 rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max, &i); 1537 } while (rc == VIRTQUEUE_READ_DESC_MORE); 1538 1539 if (rc == VIRTQUEUE_READ_DESC_ERROR) { 1540 goto err; 1541 } 1542 1543 if (desc_cache == &indirect_desc_cache) { 1544 address_space_cache_destroy(&indirect_desc_cache); 1545 total_bufs++; 1546 } else { 1547 total_bufs = num_bufs; 1548 } 1549 } 1550 1551 if (rc < 0) { 1552 goto err; 1553 } 1554 1555 done: 1556 address_space_cache_destroy(&indirect_desc_cache); 1557 if (in_bytes) { 1558 *in_bytes = in_total; 1559 } 1560 if (out_bytes) { 1561 *out_bytes = out_total; 1562 } 1563 return; 1564 1565 err: 1566 in_total = out_total = 0; 1567 goto done; 1568 } 1569 1570 static int virtqueue_packed_read_next_desc(VirtQueue *vq, 1571 VRingPackedDesc *desc, 1572 MemoryRegionCache 1573 *desc_cache, 1574 unsigned int max, 1575 unsigned int *next, 1576 bool indirect) 1577 { 1578 /* If this descriptor says it doesn't chain, we're done. */ 1579 if (!indirect && !(desc->flags & VRING_DESC_F_NEXT)) { 1580 return VIRTQUEUE_READ_DESC_DONE; 1581 } 1582 1583 ++*next; 1584 if (*next == max) { 1585 if (indirect) { 1586 return VIRTQUEUE_READ_DESC_DONE; 1587 } else { 1588 (*next) -= vq->vring.num; 1589 } 1590 } 1591 1592 vring_packed_desc_read(vq->vdev, desc, desc_cache, *next, false); 1593 return VIRTQUEUE_READ_DESC_MORE; 1594 } 1595 1596 /* Called within rcu_read_lock(). */ 1597 static void virtqueue_packed_get_avail_bytes(VirtQueue *vq, 1598 unsigned int *in_bytes, 1599 unsigned int *out_bytes, 1600 unsigned max_in_bytes, 1601 unsigned max_out_bytes, 1602 VRingMemoryRegionCaches *caches) 1603 { 1604 VirtIODevice *vdev = vq->vdev; 1605 unsigned int max, idx; 1606 unsigned int total_bufs, in_total, out_total; 1607 MemoryRegionCache *desc_cache; 1608 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 1609 int64_t len = 0; 1610 VRingPackedDesc desc; 1611 bool wrap_counter; 1612 1613 idx = vq->last_avail_idx; 1614 wrap_counter = vq->last_avail_wrap_counter; 1615 total_bufs = in_total = out_total = 0; 1616 1617 max = vq->vring.num; 1618 1619 for (;;) { 1620 unsigned int num_bufs = total_bufs; 1621 unsigned int i = idx; 1622 int rc; 1623 1624 desc_cache = &caches->desc; 1625 vring_packed_desc_read(vdev, &desc, desc_cache, idx, true); 1626 if (!is_desc_avail(desc.flags, wrap_counter)) { 1627 break; 1628 } 1629 1630 if (desc.flags & VRING_DESC_F_INDIRECT) { 1631 if (desc.len % sizeof(VRingPackedDesc)) { 1632 virtio_error(vdev, "Invalid size for indirect buffer table"); 1633 goto err; 1634 } 1635 1636 /* If we've got too many, that implies a descriptor loop. */ 1637 if (num_bufs >= max) { 1638 virtio_error(vdev, "Looped descriptor"); 1639 goto err; 1640 } 1641 1642 /* loop over the indirect descriptor table */ 1643 len = address_space_cache_init(&indirect_desc_cache, 1644 vdev->dma_as, 1645 desc.addr, desc.len, false); 1646 desc_cache = &indirect_desc_cache; 1647 if (len < desc.len) { 1648 virtio_error(vdev, "Cannot map indirect buffer"); 1649 goto err; 1650 } 1651 1652 max = desc.len / sizeof(VRingPackedDesc); 1653 num_bufs = i = 0; 1654 vring_packed_desc_read(vdev, &desc, desc_cache, i, false); 1655 } 1656 1657 do { 1658 /* If we've got too many, that implies a descriptor loop. */ 1659 if (++num_bufs > max) { 1660 virtio_error(vdev, "Looped descriptor"); 1661 goto err; 1662 } 1663 1664 if (desc.flags & VRING_DESC_F_WRITE) { 1665 in_total += desc.len; 1666 } else { 1667 out_total += desc.len; 1668 } 1669 if (in_total >= max_in_bytes && out_total >= max_out_bytes) { 1670 goto done; 1671 } 1672 1673 rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max, 1674 &i, desc_cache == 1675 &indirect_desc_cache); 1676 } while (rc == VIRTQUEUE_READ_DESC_MORE); 1677 1678 if (desc_cache == &indirect_desc_cache) { 1679 address_space_cache_destroy(&indirect_desc_cache); 1680 total_bufs++; 1681 idx++; 1682 } else { 1683 idx += num_bufs - total_bufs; 1684 total_bufs = num_bufs; 1685 } 1686 1687 if (idx >= vq->vring.num) { 1688 idx -= vq->vring.num; 1689 wrap_counter ^= 1; 1690 } 1691 } 1692 1693 /* Record the index and wrap counter for a kick we want */ 1694 vq->shadow_avail_idx = idx; 1695 vq->shadow_avail_wrap_counter = wrap_counter; 1696 done: 1697 address_space_cache_destroy(&indirect_desc_cache); 1698 if (in_bytes) { 1699 *in_bytes = in_total; 1700 } 1701 if (out_bytes) { 1702 *out_bytes = out_total; 1703 } 1704 return; 1705 1706 err: 1707 in_total = out_total = 0; 1708 goto done; 1709 } 1710 1711 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes, 1712 unsigned int *out_bytes, 1713 unsigned max_in_bytes, unsigned max_out_bytes) 1714 { 1715 uint16_t desc_size; 1716 VRingMemoryRegionCaches *caches; 1717 1718 RCU_READ_LOCK_GUARD(); 1719 1720 if (unlikely(!vq->vring.desc)) { 1721 goto err; 1722 } 1723 1724 caches = vring_get_region_caches(vq); 1725 if (!caches) { 1726 goto err; 1727 } 1728 1729 desc_size = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ? 1730 sizeof(VRingPackedDesc) : sizeof(VRingDesc); 1731 if (caches->desc.len < vq->vring.num * desc_size) { 1732 virtio_error(vq->vdev, "Cannot map descriptor ring"); 1733 goto err; 1734 } 1735 1736 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1737 virtqueue_packed_get_avail_bytes(vq, in_bytes, out_bytes, 1738 max_in_bytes, max_out_bytes, 1739 caches); 1740 } else { 1741 virtqueue_split_get_avail_bytes(vq, in_bytes, out_bytes, 1742 max_in_bytes, max_out_bytes, 1743 caches); 1744 } 1745 1746 return; 1747 err: 1748 if (in_bytes) { 1749 *in_bytes = 0; 1750 } 1751 if (out_bytes) { 1752 *out_bytes = 0; 1753 } 1754 } 1755 1756 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes, 1757 unsigned int out_bytes) 1758 { 1759 unsigned int in_total, out_total; 1760 1761 virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes); 1762 return in_bytes <= in_total && out_bytes <= out_total; 1763 } 1764 1765 static bool virtqueue_map_desc(VirtIODevice *vdev, unsigned int *p_num_sg, 1766 hwaddr *addr, struct iovec *iov, 1767 unsigned int max_num_sg, bool is_write, 1768 hwaddr pa, size_t sz) 1769 { 1770 bool ok = false; 1771 unsigned num_sg = *p_num_sg; 1772 assert(num_sg <= max_num_sg); 1773 1774 if (!sz) { 1775 virtio_error(vdev, "virtio: zero sized buffers are not allowed"); 1776 goto out; 1777 } 1778 1779 while (sz) { 1780 hwaddr len = sz; 1781 1782 if (num_sg == max_num_sg) { 1783 virtio_error(vdev, "virtio: too many write descriptors in " 1784 "indirect table"); 1785 goto out; 1786 } 1787 1788 iov[num_sg].iov_base = dma_memory_map(vdev->dma_as, pa, &len, 1789 is_write ? 1790 DMA_DIRECTION_FROM_DEVICE : 1791 DMA_DIRECTION_TO_DEVICE, 1792 MEMTXATTRS_UNSPECIFIED); 1793 if (!iov[num_sg].iov_base) { 1794 virtio_error(vdev, "virtio: bogus descriptor or out of resources"); 1795 goto out; 1796 } 1797 1798 iov[num_sg].iov_len = len; 1799 addr[num_sg] = pa; 1800 1801 sz -= len; 1802 pa += len; 1803 num_sg++; 1804 } 1805 ok = true; 1806 1807 out: 1808 *p_num_sg = num_sg; 1809 return ok; 1810 } 1811 1812 /* Only used by error code paths before we have a VirtQueueElement (therefore 1813 * virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to 1814 * yet. 1815 */ 1816 static void virtqueue_undo_map_desc(unsigned int out_num, unsigned int in_num, 1817 struct iovec *iov) 1818 { 1819 unsigned int i; 1820 1821 for (i = 0; i < out_num + in_num; i++) { 1822 int is_write = i >= out_num; 1823 1824 cpu_physical_memory_unmap(iov->iov_base, iov->iov_len, is_write, 0); 1825 iov++; 1826 } 1827 } 1828 1829 static void virtqueue_map_iovec(VirtIODevice *vdev, struct iovec *sg, 1830 hwaddr *addr, unsigned int num_sg, 1831 bool is_write) 1832 { 1833 unsigned int i; 1834 hwaddr len; 1835 1836 for (i = 0; i < num_sg; i++) { 1837 len = sg[i].iov_len; 1838 sg[i].iov_base = dma_memory_map(vdev->dma_as, 1839 addr[i], &len, is_write ? 1840 DMA_DIRECTION_FROM_DEVICE : 1841 DMA_DIRECTION_TO_DEVICE, 1842 MEMTXATTRS_UNSPECIFIED); 1843 if (!sg[i].iov_base) { 1844 error_report("virtio: error trying to map MMIO memory"); 1845 exit(1); 1846 } 1847 if (len != sg[i].iov_len) { 1848 error_report("virtio: unexpected memory split"); 1849 exit(1); 1850 } 1851 } 1852 } 1853 1854 void virtqueue_map(VirtIODevice *vdev, VirtQueueElement *elem) 1855 { 1856 virtqueue_map_iovec(vdev, elem->in_sg, elem->in_addr, elem->in_num, true); 1857 virtqueue_map_iovec(vdev, elem->out_sg, elem->out_addr, elem->out_num, 1858 false); 1859 } 1860 1861 static void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num) 1862 { 1863 VirtQueueElement *elem; 1864 size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0])); 1865 size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]); 1866 size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]); 1867 size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0])); 1868 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]); 1869 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]); 1870 1871 assert(sz >= sizeof(VirtQueueElement)); 1872 elem = g_malloc(out_sg_end); 1873 trace_virtqueue_alloc_element(elem, sz, in_num, out_num); 1874 elem->out_num = out_num; 1875 elem->in_num = in_num; 1876 elem->in_addr = (void *)elem + in_addr_ofs; 1877 elem->out_addr = (void *)elem + out_addr_ofs; 1878 elem->in_sg = (void *)elem + in_sg_ofs; 1879 elem->out_sg = (void *)elem + out_sg_ofs; 1880 return elem; 1881 } 1882 1883 static void *virtqueue_split_pop(VirtQueue *vq, size_t sz) 1884 { 1885 unsigned int i, head, max; 1886 VRingMemoryRegionCaches *caches; 1887 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 1888 MemoryRegionCache *desc_cache; 1889 int64_t len; 1890 VirtIODevice *vdev = vq->vdev; 1891 VirtQueueElement *elem = NULL; 1892 unsigned out_num, in_num, elem_entries; 1893 hwaddr addr[VIRTQUEUE_MAX_SIZE]; 1894 struct iovec iov[VIRTQUEUE_MAX_SIZE]; 1895 VRingDesc desc; 1896 int rc; 1897 1898 RCU_READ_LOCK_GUARD(); 1899 if (virtio_queue_empty_rcu(vq)) { 1900 goto done; 1901 } 1902 /* Needed after virtio_queue_empty(), see comment in 1903 * virtqueue_num_heads(). */ 1904 smp_rmb(); 1905 1906 /* When we start there are none of either input nor output. */ 1907 out_num = in_num = elem_entries = 0; 1908 1909 max = vq->vring.num; 1910 1911 if (vq->inuse >= vq->vring.num) { 1912 virtio_error(vdev, "Virtqueue size exceeded"); 1913 goto done; 1914 } 1915 1916 if (!virtqueue_get_head(vq, vq->last_avail_idx++, &head)) { 1917 goto done; 1918 } 1919 1920 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 1921 vring_set_avail_event(vq, vq->last_avail_idx); 1922 } 1923 1924 i = head; 1925 1926 caches = vring_get_region_caches(vq); 1927 if (!caches) { 1928 virtio_error(vdev, "Region caches not initialized"); 1929 goto done; 1930 } 1931 1932 if (caches->desc.len < max * sizeof(VRingDesc)) { 1933 virtio_error(vdev, "Cannot map descriptor ring"); 1934 goto done; 1935 } 1936 1937 desc_cache = &caches->desc; 1938 vring_split_desc_read(vdev, &desc, desc_cache, i); 1939 if (desc.flags & VRING_DESC_F_INDIRECT) { 1940 if (!desc.len || (desc.len % sizeof(VRingDesc))) { 1941 virtio_error(vdev, "Invalid size for indirect buffer table"); 1942 goto done; 1943 } 1944 1945 /* loop over the indirect descriptor table */ 1946 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, 1947 desc.addr, desc.len, false); 1948 desc_cache = &indirect_desc_cache; 1949 if (len < desc.len) { 1950 virtio_error(vdev, "Cannot map indirect buffer"); 1951 goto done; 1952 } 1953 1954 max = desc.len / sizeof(VRingDesc); 1955 i = 0; 1956 vring_split_desc_read(vdev, &desc, desc_cache, i); 1957 } 1958 1959 /* Collect all the descriptors */ 1960 do { 1961 bool map_ok; 1962 1963 if (desc.flags & VRING_DESC_F_WRITE) { 1964 map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num, 1965 iov + out_num, 1966 VIRTQUEUE_MAX_SIZE - out_num, true, 1967 desc.addr, desc.len); 1968 } else { 1969 if (in_num) { 1970 virtio_error(vdev, "Incorrect order for descriptors"); 1971 goto err_undo_map; 1972 } 1973 map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov, 1974 VIRTQUEUE_MAX_SIZE, false, 1975 desc.addr, desc.len); 1976 } 1977 if (!map_ok) { 1978 goto err_undo_map; 1979 } 1980 1981 /* If we've got too many, that implies a descriptor loop. */ 1982 if (++elem_entries > max) { 1983 virtio_error(vdev, "Looped descriptor"); 1984 goto err_undo_map; 1985 } 1986 1987 rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max, &i); 1988 } while (rc == VIRTQUEUE_READ_DESC_MORE); 1989 1990 if (rc == VIRTQUEUE_READ_DESC_ERROR) { 1991 goto err_undo_map; 1992 } 1993 1994 /* Now copy what we have collected and mapped */ 1995 elem = virtqueue_alloc_element(sz, out_num, in_num); 1996 elem->index = head; 1997 elem->ndescs = 1; 1998 for (i = 0; i < out_num; i++) { 1999 elem->out_addr[i] = addr[i]; 2000 elem->out_sg[i] = iov[i]; 2001 } 2002 for (i = 0; i < in_num; i++) { 2003 elem->in_addr[i] = addr[out_num + i]; 2004 elem->in_sg[i] = iov[out_num + i]; 2005 } 2006 2007 vq->inuse++; 2008 2009 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); 2010 done: 2011 address_space_cache_destroy(&indirect_desc_cache); 2012 2013 return elem; 2014 2015 err_undo_map: 2016 virtqueue_undo_map_desc(out_num, in_num, iov); 2017 goto done; 2018 } 2019 2020 static void *virtqueue_packed_pop(VirtQueue *vq, size_t sz) 2021 { 2022 unsigned int i, max; 2023 VRingMemoryRegionCaches *caches; 2024 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 2025 MemoryRegionCache *desc_cache; 2026 int64_t len; 2027 VirtIODevice *vdev = vq->vdev; 2028 VirtQueueElement *elem = NULL; 2029 unsigned out_num, in_num, elem_entries; 2030 hwaddr addr[VIRTQUEUE_MAX_SIZE]; 2031 struct iovec iov[VIRTQUEUE_MAX_SIZE]; 2032 VRingPackedDesc desc; 2033 uint16_t id; 2034 int rc; 2035 2036 RCU_READ_LOCK_GUARD(); 2037 if (virtio_queue_packed_empty_rcu(vq)) { 2038 goto done; 2039 } 2040 2041 /* When we start there are none of either input nor output. */ 2042 out_num = in_num = elem_entries = 0; 2043 2044 max = vq->vring.num; 2045 2046 if (vq->inuse >= vq->vring.num) { 2047 virtio_error(vdev, "Virtqueue size exceeded"); 2048 goto done; 2049 } 2050 2051 i = vq->last_avail_idx; 2052 2053 caches = vring_get_region_caches(vq); 2054 if (!caches) { 2055 virtio_error(vdev, "Region caches not initialized"); 2056 goto done; 2057 } 2058 2059 if (caches->desc.len < max * sizeof(VRingDesc)) { 2060 virtio_error(vdev, "Cannot map descriptor ring"); 2061 goto done; 2062 } 2063 2064 desc_cache = &caches->desc; 2065 vring_packed_desc_read(vdev, &desc, desc_cache, i, true); 2066 id = desc.id; 2067 if (desc.flags & VRING_DESC_F_INDIRECT) { 2068 if (desc.len % sizeof(VRingPackedDesc)) { 2069 virtio_error(vdev, "Invalid size for indirect buffer table"); 2070 goto done; 2071 } 2072 2073 /* loop over the indirect descriptor table */ 2074 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, 2075 desc.addr, desc.len, false); 2076 desc_cache = &indirect_desc_cache; 2077 if (len < desc.len) { 2078 virtio_error(vdev, "Cannot map indirect buffer"); 2079 goto done; 2080 } 2081 2082 max = desc.len / sizeof(VRingPackedDesc); 2083 i = 0; 2084 vring_packed_desc_read(vdev, &desc, desc_cache, i, false); 2085 } 2086 2087 /* Collect all the descriptors */ 2088 do { 2089 bool map_ok; 2090 2091 if (desc.flags & VRING_DESC_F_WRITE) { 2092 map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num, 2093 iov + out_num, 2094 VIRTQUEUE_MAX_SIZE - out_num, true, 2095 desc.addr, desc.len); 2096 } else { 2097 if (in_num) { 2098 virtio_error(vdev, "Incorrect order for descriptors"); 2099 goto err_undo_map; 2100 } 2101 map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov, 2102 VIRTQUEUE_MAX_SIZE, false, 2103 desc.addr, desc.len); 2104 } 2105 if (!map_ok) { 2106 goto err_undo_map; 2107 } 2108 2109 /* If we've got too many, that implies a descriptor loop. */ 2110 if (++elem_entries > max) { 2111 virtio_error(vdev, "Looped descriptor"); 2112 goto err_undo_map; 2113 } 2114 2115 rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max, &i, 2116 desc_cache == 2117 &indirect_desc_cache); 2118 } while (rc == VIRTQUEUE_READ_DESC_MORE); 2119 2120 /* Now copy what we have collected and mapped */ 2121 elem = virtqueue_alloc_element(sz, out_num, in_num); 2122 for (i = 0; i < out_num; i++) { 2123 elem->out_addr[i] = addr[i]; 2124 elem->out_sg[i] = iov[i]; 2125 } 2126 for (i = 0; i < in_num; i++) { 2127 elem->in_addr[i] = addr[out_num + i]; 2128 elem->in_sg[i] = iov[out_num + i]; 2129 } 2130 2131 elem->index = id; 2132 elem->ndescs = (desc_cache == &indirect_desc_cache) ? 1 : elem_entries; 2133 vq->last_avail_idx += elem->ndescs; 2134 vq->inuse += elem->ndescs; 2135 2136 if (vq->last_avail_idx >= vq->vring.num) { 2137 vq->last_avail_idx -= vq->vring.num; 2138 vq->last_avail_wrap_counter ^= 1; 2139 } 2140 2141 vq->shadow_avail_idx = vq->last_avail_idx; 2142 vq->shadow_avail_wrap_counter = vq->last_avail_wrap_counter; 2143 2144 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); 2145 done: 2146 address_space_cache_destroy(&indirect_desc_cache); 2147 2148 return elem; 2149 2150 err_undo_map: 2151 virtqueue_undo_map_desc(out_num, in_num, iov); 2152 goto done; 2153 } 2154 2155 void *virtqueue_pop(VirtQueue *vq, size_t sz) 2156 { 2157 if (virtio_device_disabled(vq->vdev)) { 2158 return NULL; 2159 } 2160 2161 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 2162 return virtqueue_packed_pop(vq, sz); 2163 } else { 2164 return virtqueue_split_pop(vq, sz); 2165 } 2166 } 2167 2168 static unsigned int virtqueue_packed_drop_all(VirtQueue *vq) 2169 { 2170 VRingMemoryRegionCaches *caches; 2171 MemoryRegionCache *desc_cache; 2172 unsigned int dropped = 0; 2173 VirtQueueElement elem = {}; 2174 VirtIODevice *vdev = vq->vdev; 2175 VRingPackedDesc desc; 2176 2177 RCU_READ_LOCK_GUARD(); 2178 2179 caches = vring_get_region_caches(vq); 2180 if (!caches) { 2181 return 0; 2182 } 2183 2184 desc_cache = &caches->desc; 2185 2186 virtio_queue_set_notification(vq, 0); 2187 2188 while (vq->inuse < vq->vring.num) { 2189 unsigned int idx = vq->last_avail_idx; 2190 /* 2191 * works similar to virtqueue_pop but does not map buffers 2192 * and does not allocate any memory. 2193 */ 2194 vring_packed_desc_read(vdev, &desc, desc_cache, 2195 vq->last_avail_idx , true); 2196 if (!is_desc_avail(desc.flags, vq->last_avail_wrap_counter)) { 2197 break; 2198 } 2199 elem.index = desc.id; 2200 elem.ndescs = 1; 2201 while (virtqueue_packed_read_next_desc(vq, &desc, desc_cache, 2202 vq->vring.num, &idx, false)) { 2203 ++elem.ndescs; 2204 } 2205 /* 2206 * immediately push the element, nothing to unmap 2207 * as both in_num and out_num are set to 0. 2208 */ 2209 virtqueue_push(vq, &elem, 0); 2210 dropped++; 2211 vq->last_avail_idx += elem.ndescs; 2212 if (vq->last_avail_idx >= vq->vring.num) { 2213 vq->last_avail_idx -= vq->vring.num; 2214 vq->last_avail_wrap_counter ^= 1; 2215 } 2216 } 2217 2218 return dropped; 2219 } 2220 2221 static unsigned int virtqueue_split_drop_all(VirtQueue *vq) 2222 { 2223 unsigned int dropped = 0; 2224 VirtQueueElement elem = {}; 2225 VirtIODevice *vdev = vq->vdev; 2226 bool fEventIdx = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); 2227 2228 while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) { 2229 /* works similar to virtqueue_pop but does not map buffers 2230 * and does not allocate any memory */ 2231 smp_rmb(); 2232 if (!virtqueue_get_head(vq, vq->last_avail_idx, &elem.index)) { 2233 break; 2234 } 2235 vq->inuse++; 2236 vq->last_avail_idx++; 2237 if (fEventIdx) { 2238 vring_set_avail_event(vq, vq->last_avail_idx); 2239 } 2240 /* immediately push the element, nothing to unmap 2241 * as both in_num and out_num are set to 0 */ 2242 virtqueue_push(vq, &elem, 0); 2243 dropped++; 2244 } 2245 2246 return dropped; 2247 } 2248 2249 /* virtqueue_drop_all: 2250 * @vq: The #VirtQueue 2251 * Drops all queued buffers and indicates them to the guest 2252 * as if they are done. Useful when buffers can not be 2253 * processed but must be returned to the guest. 2254 */ 2255 unsigned int virtqueue_drop_all(VirtQueue *vq) 2256 { 2257 struct VirtIODevice *vdev = vq->vdev; 2258 2259 if (virtio_device_disabled(vq->vdev)) { 2260 return 0; 2261 } 2262 2263 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 2264 return virtqueue_packed_drop_all(vq); 2265 } else { 2266 return virtqueue_split_drop_all(vq); 2267 } 2268 } 2269 2270 /* Reading and writing a structure directly to QEMUFile is *awful*, but 2271 * it is what QEMU has always done by mistake. We can change it sooner 2272 * or later by bumping the version number of the affected vm states. 2273 * In the meanwhile, since the in-memory layout of VirtQueueElement 2274 * has changed, we need to marshal to and from the layout that was 2275 * used before the change. 2276 */ 2277 typedef struct VirtQueueElementOld { 2278 unsigned int index; 2279 unsigned int out_num; 2280 unsigned int in_num; 2281 hwaddr in_addr[VIRTQUEUE_MAX_SIZE]; 2282 hwaddr out_addr[VIRTQUEUE_MAX_SIZE]; 2283 struct iovec in_sg[VIRTQUEUE_MAX_SIZE]; 2284 struct iovec out_sg[VIRTQUEUE_MAX_SIZE]; 2285 } VirtQueueElementOld; 2286 2287 void *qemu_get_virtqueue_element(VirtIODevice *vdev, QEMUFile *f, size_t sz) 2288 { 2289 VirtQueueElement *elem; 2290 VirtQueueElementOld data; 2291 int i; 2292 2293 qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld)); 2294 2295 /* TODO: teach all callers that this can fail, and return failure instead 2296 * of asserting here. 2297 * This is just one thing (there are probably more) that must be 2298 * fixed before we can allow NDEBUG compilation. 2299 */ 2300 assert(ARRAY_SIZE(data.in_addr) >= data.in_num); 2301 assert(ARRAY_SIZE(data.out_addr) >= data.out_num); 2302 2303 elem = virtqueue_alloc_element(sz, data.out_num, data.in_num); 2304 elem->index = data.index; 2305 2306 for (i = 0; i < elem->in_num; i++) { 2307 elem->in_addr[i] = data.in_addr[i]; 2308 } 2309 2310 for (i = 0; i < elem->out_num; i++) { 2311 elem->out_addr[i] = data.out_addr[i]; 2312 } 2313 2314 for (i = 0; i < elem->in_num; i++) { 2315 /* Base is overwritten by virtqueue_map. */ 2316 elem->in_sg[i].iov_base = 0; 2317 elem->in_sg[i].iov_len = data.in_sg[i].iov_len; 2318 } 2319 2320 for (i = 0; i < elem->out_num; i++) { 2321 /* Base is overwritten by virtqueue_map. */ 2322 elem->out_sg[i].iov_base = 0; 2323 elem->out_sg[i].iov_len = data.out_sg[i].iov_len; 2324 } 2325 2326 if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 2327 qemu_get_be32s(f, &elem->ndescs); 2328 } 2329 2330 virtqueue_map(vdev, elem); 2331 return elem; 2332 } 2333 2334 void qemu_put_virtqueue_element(VirtIODevice *vdev, QEMUFile *f, 2335 VirtQueueElement *elem) 2336 { 2337 VirtQueueElementOld data; 2338 int i; 2339 2340 memset(&data, 0, sizeof(data)); 2341 data.index = elem->index; 2342 data.in_num = elem->in_num; 2343 data.out_num = elem->out_num; 2344 2345 for (i = 0; i < elem->in_num; i++) { 2346 data.in_addr[i] = elem->in_addr[i]; 2347 } 2348 2349 for (i = 0; i < elem->out_num; i++) { 2350 data.out_addr[i] = elem->out_addr[i]; 2351 } 2352 2353 for (i = 0; i < elem->in_num; i++) { 2354 /* Base is overwritten by virtqueue_map when loading. Do not 2355 * save it, as it would leak the QEMU address space layout. */ 2356 data.in_sg[i].iov_len = elem->in_sg[i].iov_len; 2357 } 2358 2359 for (i = 0; i < elem->out_num; i++) { 2360 /* Do not save iov_base as above. */ 2361 data.out_sg[i].iov_len = elem->out_sg[i].iov_len; 2362 } 2363 2364 if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 2365 qemu_put_be32s(f, &elem->ndescs); 2366 } 2367 2368 qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld)); 2369 } 2370 2371 /* virtio device */ 2372 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector) 2373 { 2374 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2375 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 2376 2377 if (virtio_device_disabled(vdev)) { 2378 return; 2379 } 2380 2381 if (k->notify) { 2382 k->notify(qbus->parent, vector); 2383 } 2384 } 2385 2386 void virtio_update_irq(VirtIODevice *vdev) 2387 { 2388 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); 2389 } 2390 2391 static int virtio_validate_features(VirtIODevice *vdev) 2392 { 2393 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2394 2395 if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM) && 2396 !virtio_vdev_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) { 2397 return -EFAULT; 2398 } 2399 2400 if (k->validate_features) { 2401 return k->validate_features(vdev); 2402 } else { 2403 return 0; 2404 } 2405 } 2406 2407 int virtio_set_status(VirtIODevice *vdev, uint8_t val) 2408 { 2409 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2410 trace_virtio_set_status(vdev, val); 2411 2412 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 2413 if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) && 2414 val & VIRTIO_CONFIG_S_FEATURES_OK) { 2415 int ret = virtio_validate_features(vdev); 2416 2417 if (ret) { 2418 return ret; 2419 } 2420 } 2421 } 2422 2423 if ((vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) != 2424 (val & VIRTIO_CONFIG_S_DRIVER_OK)) { 2425 virtio_set_started(vdev, val & VIRTIO_CONFIG_S_DRIVER_OK); 2426 } 2427 2428 if (k->set_status) { 2429 k->set_status(vdev, val); 2430 } 2431 vdev->status = val; 2432 2433 return 0; 2434 } 2435 2436 static enum virtio_device_endian virtio_default_endian(void) 2437 { 2438 if (target_words_bigendian()) { 2439 return VIRTIO_DEVICE_ENDIAN_BIG; 2440 } else { 2441 return VIRTIO_DEVICE_ENDIAN_LITTLE; 2442 } 2443 } 2444 2445 static enum virtio_device_endian virtio_current_cpu_endian(void) 2446 { 2447 if (cpu_virtio_is_big_endian(current_cpu)) { 2448 return VIRTIO_DEVICE_ENDIAN_BIG; 2449 } else { 2450 return VIRTIO_DEVICE_ENDIAN_LITTLE; 2451 } 2452 } 2453 2454 void virtio_reset(void *opaque) 2455 { 2456 VirtIODevice *vdev = opaque; 2457 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2458 int i; 2459 2460 virtio_set_status(vdev, 0); 2461 if (current_cpu) { 2462 /* Guest initiated reset */ 2463 vdev->device_endian = virtio_current_cpu_endian(); 2464 } else { 2465 /* System reset */ 2466 vdev->device_endian = virtio_default_endian(); 2467 } 2468 2469 if (k->reset) { 2470 k->reset(vdev); 2471 } 2472 2473 vdev->start_on_kick = false; 2474 vdev->started = false; 2475 vdev->broken = false; 2476 vdev->guest_features = 0; 2477 vdev->queue_sel = 0; 2478 vdev->status = 0; 2479 vdev->disabled = false; 2480 qatomic_set(&vdev->isr, 0); 2481 vdev->config_vector = VIRTIO_NO_VECTOR; 2482 virtio_notify_vector(vdev, vdev->config_vector); 2483 2484 for(i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2485 vdev->vq[i].vring.desc = 0; 2486 vdev->vq[i].vring.avail = 0; 2487 vdev->vq[i].vring.used = 0; 2488 vdev->vq[i].last_avail_idx = 0; 2489 vdev->vq[i].shadow_avail_idx = 0; 2490 vdev->vq[i].used_idx = 0; 2491 vdev->vq[i].last_avail_wrap_counter = true; 2492 vdev->vq[i].shadow_avail_wrap_counter = true; 2493 vdev->vq[i].used_wrap_counter = true; 2494 virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR); 2495 vdev->vq[i].signalled_used = 0; 2496 vdev->vq[i].signalled_used_valid = false; 2497 vdev->vq[i].notification = true; 2498 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default; 2499 vdev->vq[i].inuse = 0; 2500 virtio_virtqueue_reset_region_cache(&vdev->vq[i]); 2501 } 2502 } 2503 2504 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr) 2505 { 2506 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2507 uint8_t val; 2508 2509 if (addr + sizeof(val) > vdev->config_len) { 2510 return (uint32_t)-1; 2511 } 2512 2513 k->get_config(vdev, vdev->config); 2514 2515 val = ldub_p(vdev->config + addr); 2516 return val; 2517 } 2518 2519 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr) 2520 { 2521 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2522 uint16_t val; 2523 2524 if (addr + sizeof(val) > vdev->config_len) { 2525 return (uint32_t)-1; 2526 } 2527 2528 k->get_config(vdev, vdev->config); 2529 2530 val = lduw_p(vdev->config + addr); 2531 return val; 2532 } 2533 2534 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr) 2535 { 2536 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2537 uint32_t val; 2538 2539 if (addr + sizeof(val) > vdev->config_len) { 2540 return (uint32_t)-1; 2541 } 2542 2543 k->get_config(vdev, vdev->config); 2544 2545 val = ldl_p(vdev->config + addr); 2546 return val; 2547 } 2548 2549 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data) 2550 { 2551 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2552 uint8_t val = data; 2553 2554 if (addr + sizeof(val) > vdev->config_len) { 2555 return; 2556 } 2557 2558 stb_p(vdev->config + addr, val); 2559 2560 if (k->set_config) { 2561 k->set_config(vdev, vdev->config); 2562 } 2563 } 2564 2565 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data) 2566 { 2567 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2568 uint16_t val = data; 2569 2570 if (addr + sizeof(val) > vdev->config_len) { 2571 return; 2572 } 2573 2574 stw_p(vdev->config + addr, val); 2575 2576 if (k->set_config) { 2577 k->set_config(vdev, vdev->config); 2578 } 2579 } 2580 2581 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data) 2582 { 2583 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2584 uint32_t val = data; 2585 2586 if (addr + sizeof(val) > vdev->config_len) { 2587 return; 2588 } 2589 2590 stl_p(vdev->config + addr, val); 2591 2592 if (k->set_config) { 2593 k->set_config(vdev, vdev->config); 2594 } 2595 } 2596 2597 uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr) 2598 { 2599 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2600 uint8_t val; 2601 2602 if (addr + sizeof(val) > vdev->config_len) { 2603 return (uint32_t)-1; 2604 } 2605 2606 k->get_config(vdev, vdev->config); 2607 2608 val = ldub_p(vdev->config + addr); 2609 return val; 2610 } 2611 2612 uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr) 2613 { 2614 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2615 uint16_t val; 2616 2617 if (addr + sizeof(val) > vdev->config_len) { 2618 return (uint32_t)-1; 2619 } 2620 2621 k->get_config(vdev, vdev->config); 2622 2623 val = lduw_le_p(vdev->config + addr); 2624 return val; 2625 } 2626 2627 uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr) 2628 { 2629 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2630 uint32_t val; 2631 2632 if (addr + sizeof(val) > vdev->config_len) { 2633 return (uint32_t)-1; 2634 } 2635 2636 k->get_config(vdev, vdev->config); 2637 2638 val = ldl_le_p(vdev->config + addr); 2639 return val; 2640 } 2641 2642 void virtio_config_modern_writeb(VirtIODevice *vdev, 2643 uint32_t addr, uint32_t data) 2644 { 2645 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2646 uint8_t val = data; 2647 2648 if (addr + sizeof(val) > vdev->config_len) { 2649 return; 2650 } 2651 2652 stb_p(vdev->config + addr, val); 2653 2654 if (k->set_config) { 2655 k->set_config(vdev, vdev->config); 2656 } 2657 } 2658 2659 void virtio_config_modern_writew(VirtIODevice *vdev, 2660 uint32_t addr, uint32_t data) 2661 { 2662 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2663 uint16_t val = data; 2664 2665 if (addr + sizeof(val) > vdev->config_len) { 2666 return; 2667 } 2668 2669 stw_le_p(vdev->config + addr, val); 2670 2671 if (k->set_config) { 2672 k->set_config(vdev, vdev->config); 2673 } 2674 } 2675 2676 void virtio_config_modern_writel(VirtIODevice *vdev, 2677 uint32_t addr, uint32_t data) 2678 { 2679 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2680 uint32_t val = data; 2681 2682 if (addr + sizeof(val) > vdev->config_len) { 2683 return; 2684 } 2685 2686 stl_le_p(vdev->config + addr, val); 2687 2688 if (k->set_config) { 2689 k->set_config(vdev, vdev->config); 2690 } 2691 } 2692 2693 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr) 2694 { 2695 if (!vdev->vq[n].vring.num) { 2696 return; 2697 } 2698 vdev->vq[n].vring.desc = addr; 2699 virtio_queue_update_rings(vdev, n); 2700 } 2701 2702 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n) 2703 { 2704 return vdev->vq[n].vring.desc; 2705 } 2706 2707 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc, 2708 hwaddr avail, hwaddr used) 2709 { 2710 if (!vdev->vq[n].vring.num) { 2711 return; 2712 } 2713 vdev->vq[n].vring.desc = desc; 2714 vdev->vq[n].vring.avail = avail; 2715 vdev->vq[n].vring.used = used; 2716 virtio_init_region_cache(vdev, n); 2717 } 2718 2719 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num) 2720 { 2721 /* Don't allow guest to flip queue between existent and 2722 * nonexistent states, or to set it to an invalid size. 2723 */ 2724 if (!!num != !!vdev->vq[n].vring.num || 2725 num > VIRTQUEUE_MAX_SIZE || 2726 num < 0) { 2727 return; 2728 } 2729 vdev->vq[n].vring.num = num; 2730 } 2731 2732 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector) 2733 { 2734 return QLIST_FIRST(&vdev->vector_queues[vector]); 2735 } 2736 2737 VirtQueue *virtio_vector_next_queue(VirtQueue *vq) 2738 { 2739 return QLIST_NEXT(vq, node); 2740 } 2741 2742 int virtio_queue_get_num(VirtIODevice *vdev, int n) 2743 { 2744 return vdev->vq[n].vring.num; 2745 } 2746 2747 int virtio_queue_get_max_num(VirtIODevice *vdev, int n) 2748 { 2749 return vdev->vq[n].vring.num_default; 2750 } 2751 2752 int virtio_get_num_queues(VirtIODevice *vdev) 2753 { 2754 int i; 2755 2756 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2757 if (!virtio_queue_get_num(vdev, i)) { 2758 break; 2759 } 2760 } 2761 2762 return i; 2763 } 2764 2765 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align) 2766 { 2767 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2768 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 2769 2770 /* virtio-1 compliant devices cannot change the alignment */ 2771 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 2772 error_report("tried to modify queue alignment for virtio-1 device"); 2773 return; 2774 } 2775 /* Check that the transport told us it was going to do this 2776 * (so a buggy transport will immediately assert rather than 2777 * silently failing to migrate this state) 2778 */ 2779 assert(k->has_variable_vring_alignment); 2780 2781 if (align) { 2782 vdev->vq[n].vring.align = align; 2783 virtio_queue_update_rings(vdev, n); 2784 } 2785 } 2786 2787 static void virtio_queue_notify_vq(VirtQueue *vq) 2788 { 2789 if (vq->vring.desc && vq->handle_output) { 2790 VirtIODevice *vdev = vq->vdev; 2791 2792 if (unlikely(vdev->broken)) { 2793 return; 2794 } 2795 2796 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); 2797 vq->handle_output(vdev, vq); 2798 2799 if (unlikely(vdev->start_on_kick)) { 2800 virtio_set_started(vdev, true); 2801 } 2802 } 2803 } 2804 2805 void virtio_queue_notify(VirtIODevice *vdev, int n) 2806 { 2807 VirtQueue *vq = &vdev->vq[n]; 2808 2809 if (unlikely(!vq->vring.desc || vdev->broken)) { 2810 return; 2811 } 2812 2813 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); 2814 if (vq->host_notifier_enabled) { 2815 event_notifier_set(&vq->host_notifier); 2816 } else if (vq->handle_output) { 2817 vq->handle_output(vdev, vq); 2818 2819 if (unlikely(vdev->start_on_kick)) { 2820 virtio_set_started(vdev, true); 2821 } 2822 } 2823 } 2824 2825 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n) 2826 { 2827 return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector : 2828 VIRTIO_NO_VECTOR; 2829 } 2830 2831 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector) 2832 { 2833 VirtQueue *vq = &vdev->vq[n]; 2834 2835 if (n < VIRTIO_QUEUE_MAX) { 2836 if (vdev->vector_queues && 2837 vdev->vq[n].vector != VIRTIO_NO_VECTOR) { 2838 QLIST_REMOVE(vq, node); 2839 } 2840 vdev->vq[n].vector = vector; 2841 if (vdev->vector_queues && 2842 vector != VIRTIO_NO_VECTOR) { 2843 QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node); 2844 } 2845 } 2846 } 2847 2848 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size, 2849 VirtIOHandleOutput handle_output) 2850 { 2851 int i; 2852 2853 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2854 if (vdev->vq[i].vring.num == 0) 2855 break; 2856 } 2857 2858 if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE) 2859 abort(); 2860 2861 vdev->vq[i].vring.num = queue_size; 2862 vdev->vq[i].vring.num_default = queue_size; 2863 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN; 2864 vdev->vq[i].handle_output = handle_output; 2865 vdev->vq[i].used_elems = g_new0(VirtQueueElement, queue_size); 2866 2867 return &vdev->vq[i]; 2868 } 2869 2870 void virtio_delete_queue(VirtQueue *vq) 2871 { 2872 vq->vring.num = 0; 2873 vq->vring.num_default = 0; 2874 vq->handle_output = NULL; 2875 g_free(vq->used_elems); 2876 vq->used_elems = NULL; 2877 virtio_virtqueue_reset_region_cache(vq); 2878 } 2879 2880 void virtio_del_queue(VirtIODevice *vdev, int n) 2881 { 2882 if (n < 0 || n >= VIRTIO_QUEUE_MAX) { 2883 abort(); 2884 } 2885 2886 virtio_delete_queue(&vdev->vq[n]); 2887 } 2888 2889 static void virtio_set_isr(VirtIODevice *vdev, int value) 2890 { 2891 uint8_t old = qatomic_read(&vdev->isr); 2892 2893 /* Do not write ISR if it does not change, so that its cacheline remains 2894 * shared in the common case where the guest does not read it. 2895 */ 2896 if ((old & value) != value) { 2897 qatomic_or(&vdev->isr, value); 2898 } 2899 } 2900 2901 /* Called within rcu_read_lock(). */ 2902 static bool virtio_split_should_notify(VirtIODevice *vdev, VirtQueue *vq) 2903 { 2904 uint16_t old, new; 2905 bool v; 2906 /* We need to expose used array entries before checking used event. */ 2907 smp_mb(); 2908 /* Always notify when queue is empty (when feature acknowledge) */ 2909 if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) && 2910 !vq->inuse && virtio_queue_empty(vq)) { 2911 return true; 2912 } 2913 2914 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 2915 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT); 2916 } 2917 2918 v = vq->signalled_used_valid; 2919 vq->signalled_used_valid = true; 2920 old = vq->signalled_used; 2921 new = vq->signalled_used = vq->used_idx; 2922 return !v || vring_need_event(vring_get_used_event(vq), new, old); 2923 } 2924 2925 static bool vring_packed_need_event(VirtQueue *vq, bool wrap, 2926 uint16_t off_wrap, uint16_t new, 2927 uint16_t old) 2928 { 2929 int off = off_wrap & ~(1 << 15); 2930 2931 if (wrap != off_wrap >> 15) { 2932 off -= vq->vring.num; 2933 } 2934 2935 return vring_need_event(off, new, old); 2936 } 2937 2938 /* Called within rcu_read_lock(). */ 2939 static bool virtio_packed_should_notify(VirtIODevice *vdev, VirtQueue *vq) 2940 { 2941 VRingPackedDescEvent e; 2942 uint16_t old, new; 2943 bool v; 2944 VRingMemoryRegionCaches *caches; 2945 2946 caches = vring_get_region_caches(vq); 2947 if (!caches) { 2948 return false; 2949 } 2950 2951 vring_packed_event_read(vdev, &caches->avail, &e); 2952 2953 old = vq->signalled_used; 2954 new = vq->signalled_used = vq->used_idx; 2955 v = vq->signalled_used_valid; 2956 vq->signalled_used_valid = true; 2957 2958 if (e.flags == VRING_PACKED_EVENT_FLAG_DISABLE) { 2959 return false; 2960 } else if (e.flags == VRING_PACKED_EVENT_FLAG_ENABLE) { 2961 return true; 2962 } 2963 2964 return !v || vring_packed_need_event(vq, vq->used_wrap_counter, 2965 e.off_wrap, new, old); 2966 } 2967 2968 /* Called within rcu_read_lock(). */ 2969 static bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq) 2970 { 2971 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 2972 return virtio_packed_should_notify(vdev, vq); 2973 } else { 2974 return virtio_split_should_notify(vdev, vq); 2975 } 2976 } 2977 2978 void virtio_notify_irqfd(VirtIODevice *vdev, VirtQueue *vq) 2979 { 2980 WITH_RCU_READ_LOCK_GUARD() { 2981 if (!virtio_should_notify(vdev, vq)) { 2982 return; 2983 } 2984 } 2985 2986 trace_virtio_notify_irqfd(vdev, vq); 2987 2988 /* 2989 * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but 2990 * windows drivers included in virtio-win 1.8.0 (circa 2015) are 2991 * incorrectly polling this bit during crashdump and hibernation 2992 * in MSI mode, causing a hang if this bit is never updated. 2993 * Recent releases of Windows do not really shut down, but rather 2994 * log out and hibernate to make the next startup faster. Hence, 2995 * this manifested as a more serious hang during shutdown with 2996 * 2997 * Next driver release from 2016 fixed this problem, so working around it 2998 * is not a must, but it's easy to do so let's do it here. 2999 * 3000 * Note: it's safe to update ISR from any thread as it was switched 3001 * to an atomic operation. 3002 */ 3003 virtio_set_isr(vq->vdev, 0x1); 3004 event_notifier_set(&vq->guest_notifier); 3005 } 3006 3007 static void virtio_irq(VirtQueue *vq) 3008 { 3009 virtio_set_isr(vq->vdev, 0x1); 3010 virtio_notify_vector(vq->vdev, vq->vector); 3011 } 3012 3013 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq) 3014 { 3015 WITH_RCU_READ_LOCK_GUARD() { 3016 if (!virtio_should_notify(vdev, vq)) { 3017 return; 3018 } 3019 } 3020 3021 trace_virtio_notify(vdev, vq); 3022 virtio_irq(vq); 3023 } 3024 3025 void virtio_notify_config(VirtIODevice *vdev) 3026 { 3027 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 3028 return; 3029 3030 virtio_set_isr(vdev, 0x3); 3031 vdev->generation++; 3032 virtio_notify_vector(vdev, vdev->config_vector); 3033 } 3034 3035 static bool virtio_device_endian_needed(void *opaque) 3036 { 3037 VirtIODevice *vdev = opaque; 3038 3039 assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN); 3040 if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 3041 return vdev->device_endian != virtio_default_endian(); 3042 } 3043 /* Devices conforming to VIRTIO 1.0 or later are always LE. */ 3044 return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE; 3045 } 3046 3047 static bool virtio_64bit_features_needed(void *opaque) 3048 { 3049 VirtIODevice *vdev = opaque; 3050 3051 return (vdev->host_features >> 32) != 0; 3052 } 3053 3054 static bool virtio_virtqueue_needed(void *opaque) 3055 { 3056 VirtIODevice *vdev = opaque; 3057 3058 return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1); 3059 } 3060 3061 static bool virtio_packed_virtqueue_needed(void *opaque) 3062 { 3063 VirtIODevice *vdev = opaque; 3064 3065 return virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED); 3066 } 3067 3068 static bool virtio_ringsize_needed(void *opaque) 3069 { 3070 VirtIODevice *vdev = opaque; 3071 int i; 3072 3073 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 3074 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) { 3075 return true; 3076 } 3077 } 3078 return false; 3079 } 3080 3081 static bool virtio_extra_state_needed(void *opaque) 3082 { 3083 VirtIODevice *vdev = opaque; 3084 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3085 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3086 3087 return k->has_extra_state && 3088 k->has_extra_state(qbus->parent); 3089 } 3090 3091 static bool virtio_broken_needed(void *opaque) 3092 { 3093 VirtIODevice *vdev = opaque; 3094 3095 return vdev->broken; 3096 } 3097 3098 static bool virtio_started_needed(void *opaque) 3099 { 3100 VirtIODevice *vdev = opaque; 3101 3102 return vdev->started; 3103 } 3104 3105 static bool virtio_disabled_needed(void *opaque) 3106 { 3107 VirtIODevice *vdev = opaque; 3108 3109 return vdev->disabled; 3110 } 3111 3112 static const VMStateDescription vmstate_virtqueue = { 3113 .name = "virtqueue_state", 3114 .version_id = 1, 3115 .minimum_version_id = 1, 3116 .fields = (VMStateField[]) { 3117 VMSTATE_UINT64(vring.avail, struct VirtQueue), 3118 VMSTATE_UINT64(vring.used, struct VirtQueue), 3119 VMSTATE_END_OF_LIST() 3120 } 3121 }; 3122 3123 static const VMStateDescription vmstate_packed_virtqueue = { 3124 .name = "packed_virtqueue_state", 3125 .version_id = 1, 3126 .minimum_version_id = 1, 3127 .fields = (VMStateField[]) { 3128 VMSTATE_UINT16(last_avail_idx, struct VirtQueue), 3129 VMSTATE_BOOL(last_avail_wrap_counter, struct VirtQueue), 3130 VMSTATE_UINT16(used_idx, struct VirtQueue), 3131 VMSTATE_BOOL(used_wrap_counter, struct VirtQueue), 3132 VMSTATE_UINT32(inuse, struct VirtQueue), 3133 VMSTATE_END_OF_LIST() 3134 } 3135 }; 3136 3137 static const VMStateDescription vmstate_virtio_virtqueues = { 3138 .name = "virtio/virtqueues", 3139 .version_id = 1, 3140 .minimum_version_id = 1, 3141 .needed = &virtio_virtqueue_needed, 3142 .fields = (VMStateField[]) { 3143 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, 3144 VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue), 3145 VMSTATE_END_OF_LIST() 3146 } 3147 }; 3148 3149 static const VMStateDescription vmstate_virtio_packed_virtqueues = { 3150 .name = "virtio/packed_virtqueues", 3151 .version_id = 1, 3152 .minimum_version_id = 1, 3153 .needed = &virtio_packed_virtqueue_needed, 3154 .fields = (VMStateField[]) { 3155 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, 3156 VIRTIO_QUEUE_MAX, 0, vmstate_packed_virtqueue, VirtQueue), 3157 VMSTATE_END_OF_LIST() 3158 } 3159 }; 3160 3161 static const VMStateDescription vmstate_ringsize = { 3162 .name = "ringsize_state", 3163 .version_id = 1, 3164 .minimum_version_id = 1, 3165 .fields = (VMStateField[]) { 3166 VMSTATE_UINT32(vring.num_default, struct VirtQueue), 3167 VMSTATE_END_OF_LIST() 3168 } 3169 }; 3170 3171 static const VMStateDescription vmstate_virtio_ringsize = { 3172 .name = "virtio/ringsize", 3173 .version_id = 1, 3174 .minimum_version_id = 1, 3175 .needed = &virtio_ringsize_needed, 3176 .fields = (VMStateField[]) { 3177 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, 3178 VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue), 3179 VMSTATE_END_OF_LIST() 3180 } 3181 }; 3182 3183 static int get_extra_state(QEMUFile *f, void *pv, size_t size, 3184 const VMStateField *field) 3185 { 3186 VirtIODevice *vdev = pv; 3187 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3188 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3189 3190 if (!k->load_extra_state) { 3191 return -1; 3192 } else { 3193 return k->load_extra_state(qbus->parent, f); 3194 } 3195 } 3196 3197 static int put_extra_state(QEMUFile *f, void *pv, size_t size, 3198 const VMStateField *field, JSONWriter *vmdesc) 3199 { 3200 VirtIODevice *vdev = pv; 3201 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3202 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3203 3204 k->save_extra_state(qbus->parent, f); 3205 return 0; 3206 } 3207 3208 static const VMStateInfo vmstate_info_extra_state = { 3209 .name = "virtqueue_extra_state", 3210 .get = get_extra_state, 3211 .put = put_extra_state, 3212 }; 3213 3214 static const VMStateDescription vmstate_virtio_extra_state = { 3215 .name = "virtio/extra_state", 3216 .version_id = 1, 3217 .minimum_version_id = 1, 3218 .needed = &virtio_extra_state_needed, 3219 .fields = (VMStateField[]) { 3220 { 3221 .name = "extra_state", 3222 .version_id = 0, 3223 .field_exists = NULL, 3224 .size = 0, 3225 .info = &vmstate_info_extra_state, 3226 .flags = VMS_SINGLE, 3227 .offset = 0, 3228 }, 3229 VMSTATE_END_OF_LIST() 3230 } 3231 }; 3232 3233 static const VMStateDescription vmstate_virtio_device_endian = { 3234 .name = "virtio/device_endian", 3235 .version_id = 1, 3236 .minimum_version_id = 1, 3237 .needed = &virtio_device_endian_needed, 3238 .fields = (VMStateField[]) { 3239 VMSTATE_UINT8(device_endian, VirtIODevice), 3240 VMSTATE_END_OF_LIST() 3241 } 3242 }; 3243 3244 static const VMStateDescription vmstate_virtio_64bit_features = { 3245 .name = "virtio/64bit_features", 3246 .version_id = 1, 3247 .minimum_version_id = 1, 3248 .needed = &virtio_64bit_features_needed, 3249 .fields = (VMStateField[]) { 3250 VMSTATE_UINT64(guest_features, VirtIODevice), 3251 VMSTATE_END_OF_LIST() 3252 } 3253 }; 3254 3255 static const VMStateDescription vmstate_virtio_broken = { 3256 .name = "virtio/broken", 3257 .version_id = 1, 3258 .minimum_version_id = 1, 3259 .needed = &virtio_broken_needed, 3260 .fields = (VMStateField[]) { 3261 VMSTATE_BOOL(broken, VirtIODevice), 3262 VMSTATE_END_OF_LIST() 3263 } 3264 }; 3265 3266 static const VMStateDescription vmstate_virtio_started = { 3267 .name = "virtio/started", 3268 .version_id = 1, 3269 .minimum_version_id = 1, 3270 .needed = &virtio_started_needed, 3271 .fields = (VMStateField[]) { 3272 VMSTATE_BOOL(started, VirtIODevice), 3273 VMSTATE_END_OF_LIST() 3274 } 3275 }; 3276 3277 static const VMStateDescription vmstate_virtio_disabled = { 3278 .name = "virtio/disabled", 3279 .version_id = 1, 3280 .minimum_version_id = 1, 3281 .needed = &virtio_disabled_needed, 3282 .fields = (VMStateField[]) { 3283 VMSTATE_BOOL(disabled, VirtIODevice), 3284 VMSTATE_END_OF_LIST() 3285 } 3286 }; 3287 3288 static const VMStateDescription vmstate_virtio = { 3289 .name = "virtio", 3290 .version_id = 1, 3291 .minimum_version_id = 1, 3292 .fields = (VMStateField[]) { 3293 VMSTATE_END_OF_LIST() 3294 }, 3295 .subsections = (const VMStateDescription*[]) { 3296 &vmstate_virtio_device_endian, 3297 &vmstate_virtio_64bit_features, 3298 &vmstate_virtio_virtqueues, 3299 &vmstate_virtio_ringsize, 3300 &vmstate_virtio_broken, 3301 &vmstate_virtio_extra_state, 3302 &vmstate_virtio_started, 3303 &vmstate_virtio_packed_virtqueues, 3304 &vmstate_virtio_disabled, 3305 NULL 3306 } 3307 }; 3308 3309 int virtio_save(VirtIODevice *vdev, QEMUFile *f) 3310 { 3311 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3312 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3313 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 3314 uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff); 3315 int i; 3316 3317 if (k->save_config) { 3318 k->save_config(qbus->parent, f); 3319 } 3320 3321 qemu_put_8s(f, &vdev->status); 3322 qemu_put_8s(f, &vdev->isr); 3323 qemu_put_be16s(f, &vdev->queue_sel); 3324 qemu_put_be32s(f, &guest_features_lo); 3325 qemu_put_be32(f, vdev->config_len); 3326 qemu_put_buffer(f, vdev->config, vdev->config_len); 3327 3328 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 3329 if (vdev->vq[i].vring.num == 0) 3330 break; 3331 } 3332 3333 qemu_put_be32(f, i); 3334 3335 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 3336 if (vdev->vq[i].vring.num == 0) 3337 break; 3338 3339 qemu_put_be32(f, vdev->vq[i].vring.num); 3340 if (k->has_variable_vring_alignment) { 3341 qemu_put_be32(f, vdev->vq[i].vring.align); 3342 } 3343 /* 3344 * Save desc now, the rest of the ring addresses are saved in 3345 * subsections for VIRTIO-1 devices. 3346 */ 3347 qemu_put_be64(f, vdev->vq[i].vring.desc); 3348 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx); 3349 if (k->save_queue) { 3350 k->save_queue(qbus->parent, i, f); 3351 } 3352 } 3353 3354 if (vdc->save != NULL) { 3355 vdc->save(vdev, f); 3356 } 3357 3358 if (vdc->vmsd) { 3359 int ret = vmstate_save_state(f, vdc->vmsd, vdev, NULL); 3360 if (ret) { 3361 return ret; 3362 } 3363 } 3364 3365 /* Subsections */ 3366 return vmstate_save_state(f, &vmstate_virtio, vdev, NULL); 3367 } 3368 3369 /* A wrapper for use as a VMState .put function */ 3370 static int virtio_device_put(QEMUFile *f, void *opaque, size_t size, 3371 const VMStateField *field, JSONWriter *vmdesc) 3372 { 3373 return virtio_save(VIRTIO_DEVICE(opaque), f); 3374 } 3375 3376 /* A wrapper for use as a VMState .get function */ 3377 static int virtio_device_get(QEMUFile *f, void *opaque, size_t size, 3378 const VMStateField *field) 3379 { 3380 VirtIODevice *vdev = VIRTIO_DEVICE(opaque); 3381 DeviceClass *dc = DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev)); 3382 3383 return virtio_load(vdev, f, dc->vmsd->version_id); 3384 } 3385 3386 const VMStateInfo virtio_vmstate_info = { 3387 .name = "virtio", 3388 .get = virtio_device_get, 3389 .put = virtio_device_put, 3390 }; 3391 3392 static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val) 3393 { 3394 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 3395 bool bad = (val & ~(vdev->host_features)) != 0; 3396 3397 val &= vdev->host_features; 3398 if (k->set_features) { 3399 k->set_features(vdev, val); 3400 } 3401 vdev->guest_features = val; 3402 return bad ? -1 : 0; 3403 } 3404 3405 int virtio_set_features(VirtIODevice *vdev, uint64_t val) 3406 { 3407 int ret; 3408 /* 3409 * The driver must not attempt to set features after feature negotiation 3410 * has finished. 3411 */ 3412 if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) { 3413 return -EINVAL; 3414 } 3415 3416 if (val & (1ull << VIRTIO_F_BAD_FEATURE)) { 3417 qemu_log_mask(LOG_GUEST_ERROR, 3418 "%s: guest driver for %s has enabled UNUSED(30) feature bit!\n", 3419 __func__, vdev->name); 3420 } 3421 3422 ret = virtio_set_features_nocheck(vdev, val); 3423 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 3424 /* VIRTIO_RING_F_EVENT_IDX changes the size of the caches. */ 3425 int i; 3426 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 3427 if (vdev->vq[i].vring.num != 0) { 3428 virtio_init_region_cache(vdev, i); 3429 } 3430 } 3431 } 3432 if (!ret) { 3433 if (!virtio_device_started(vdev, vdev->status) && 3434 !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 3435 vdev->start_on_kick = true; 3436 } 3437 } 3438 return ret; 3439 } 3440 3441 size_t virtio_get_config_size(const VirtIOConfigSizeParams *params, 3442 uint64_t host_features) 3443 { 3444 size_t config_size = params->min_size; 3445 const VirtIOFeature *feature_sizes = params->feature_sizes; 3446 size_t i; 3447 3448 for (i = 0; feature_sizes[i].flags != 0; i++) { 3449 if (host_features & feature_sizes[i].flags) { 3450 config_size = MAX(feature_sizes[i].end, config_size); 3451 } 3452 } 3453 3454 assert(config_size <= params->max_size); 3455 return config_size; 3456 } 3457 3458 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id) 3459 { 3460 int i, ret; 3461 int32_t config_len; 3462 uint32_t num; 3463 uint32_t features; 3464 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3465 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3466 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 3467 3468 /* 3469 * We poison the endianness to ensure it does not get used before 3470 * subsections have been loaded. 3471 */ 3472 vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN; 3473 3474 if (k->load_config) { 3475 ret = k->load_config(qbus->parent, f); 3476 if (ret) 3477 return ret; 3478 } 3479 3480 qemu_get_8s(f, &vdev->status); 3481 qemu_get_8s(f, &vdev->isr); 3482 qemu_get_be16s(f, &vdev->queue_sel); 3483 if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) { 3484 return -1; 3485 } 3486 qemu_get_be32s(f, &features); 3487 3488 /* 3489 * Temporarily set guest_features low bits - needed by 3490 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS 3491 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ. 3492 * 3493 * Note: devices should always test host features in future - don't create 3494 * new dependencies like this. 3495 */ 3496 vdev->guest_features = features; 3497 3498 config_len = qemu_get_be32(f); 3499 3500 /* 3501 * There are cases where the incoming config can be bigger or smaller 3502 * than what we have; so load what we have space for, and skip 3503 * any excess that's in the stream. 3504 */ 3505 qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len)); 3506 3507 while (config_len > vdev->config_len) { 3508 qemu_get_byte(f); 3509 config_len--; 3510 } 3511 3512 num = qemu_get_be32(f); 3513 3514 if (num > VIRTIO_QUEUE_MAX) { 3515 error_report("Invalid number of virtqueues: 0x%x", num); 3516 return -1; 3517 } 3518 3519 for (i = 0; i < num; i++) { 3520 vdev->vq[i].vring.num = qemu_get_be32(f); 3521 if (k->has_variable_vring_alignment) { 3522 vdev->vq[i].vring.align = qemu_get_be32(f); 3523 } 3524 vdev->vq[i].vring.desc = qemu_get_be64(f); 3525 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx); 3526 vdev->vq[i].signalled_used_valid = false; 3527 vdev->vq[i].notification = true; 3528 3529 if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) { 3530 error_report("VQ %d address 0x0 " 3531 "inconsistent with Host index 0x%x", 3532 i, vdev->vq[i].last_avail_idx); 3533 return -1; 3534 } 3535 if (k->load_queue) { 3536 ret = k->load_queue(qbus->parent, i, f); 3537 if (ret) 3538 return ret; 3539 } 3540 } 3541 3542 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); 3543 3544 if (vdc->load != NULL) { 3545 ret = vdc->load(vdev, f, version_id); 3546 if (ret) { 3547 return ret; 3548 } 3549 } 3550 3551 if (vdc->vmsd) { 3552 ret = vmstate_load_state(f, vdc->vmsd, vdev, version_id); 3553 if (ret) { 3554 return ret; 3555 } 3556 } 3557 3558 /* Subsections */ 3559 ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1); 3560 if (ret) { 3561 return ret; 3562 } 3563 3564 if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) { 3565 vdev->device_endian = virtio_default_endian(); 3566 } 3567 3568 if (virtio_64bit_features_needed(vdev)) { 3569 /* 3570 * Subsection load filled vdev->guest_features. Run them 3571 * through virtio_set_features to sanity-check them against 3572 * host_features. 3573 */ 3574 uint64_t features64 = vdev->guest_features; 3575 if (virtio_set_features_nocheck(vdev, features64) < 0) { 3576 error_report("Features 0x%" PRIx64 " unsupported. " 3577 "Allowed features: 0x%" PRIx64, 3578 features64, vdev->host_features); 3579 return -1; 3580 } 3581 } else { 3582 if (virtio_set_features_nocheck(vdev, features) < 0) { 3583 error_report("Features 0x%x unsupported. " 3584 "Allowed features: 0x%" PRIx64, 3585 features, vdev->host_features); 3586 return -1; 3587 } 3588 } 3589 3590 if (!virtio_device_started(vdev, vdev->status) && 3591 !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 3592 vdev->start_on_kick = true; 3593 } 3594 3595 RCU_READ_LOCK_GUARD(); 3596 for (i = 0; i < num; i++) { 3597 if (vdev->vq[i].vring.desc) { 3598 uint16_t nheads; 3599 3600 /* 3601 * VIRTIO-1 devices migrate desc, used, and avail ring addresses so 3602 * only the region cache needs to be set up. Legacy devices need 3603 * to calculate used and avail ring addresses based on the desc 3604 * address. 3605 */ 3606 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 3607 virtio_init_region_cache(vdev, i); 3608 } else { 3609 virtio_queue_update_rings(vdev, i); 3610 } 3611 3612 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3613 vdev->vq[i].shadow_avail_idx = vdev->vq[i].last_avail_idx; 3614 vdev->vq[i].shadow_avail_wrap_counter = 3615 vdev->vq[i].last_avail_wrap_counter; 3616 continue; 3617 } 3618 3619 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx; 3620 /* Check it isn't doing strange things with descriptor numbers. */ 3621 if (nheads > vdev->vq[i].vring.num) { 3622 virtio_error(vdev, "VQ %d size 0x%x Guest index 0x%x " 3623 "inconsistent with Host index 0x%x: delta 0x%x", 3624 i, vdev->vq[i].vring.num, 3625 vring_avail_idx(&vdev->vq[i]), 3626 vdev->vq[i].last_avail_idx, nheads); 3627 vdev->vq[i].used_idx = 0; 3628 vdev->vq[i].shadow_avail_idx = 0; 3629 vdev->vq[i].inuse = 0; 3630 continue; 3631 } 3632 vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]); 3633 vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]); 3634 3635 /* 3636 * Some devices migrate VirtQueueElements that have been popped 3637 * from the avail ring but not yet returned to the used ring. 3638 * Since max ring size < UINT16_MAX it's safe to use modulo 3639 * UINT16_MAX + 1 subtraction. 3640 */ 3641 vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx - 3642 vdev->vq[i].used_idx); 3643 if (vdev->vq[i].inuse > vdev->vq[i].vring.num) { 3644 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - " 3645 "used_idx 0x%x", 3646 i, vdev->vq[i].vring.num, 3647 vdev->vq[i].last_avail_idx, 3648 vdev->vq[i].used_idx); 3649 return -1; 3650 } 3651 } 3652 } 3653 3654 if (vdc->post_load) { 3655 ret = vdc->post_load(vdev); 3656 if (ret) { 3657 return ret; 3658 } 3659 } 3660 3661 return 0; 3662 } 3663 3664 void virtio_cleanup(VirtIODevice *vdev) 3665 { 3666 qemu_del_vm_change_state_handler(vdev->vmstate); 3667 } 3668 3669 static void virtio_vmstate_change(void *opaque, bool running, RunState state) 3670 { 3671 VirtIODevice *vdev = opaque; 3672 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3673 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3674 bool backend_run = running && virtio_device_started(vdev, vdev->status); 3675 vdev->vm_running = running; 3676 3677 if (backend_run) { 3678 virtio_set_status(vdev, vdev->status); 3679 } 3680 3681 if (k->vmstate_change) { 3682 k->vmstate_change(qbus->parent, backend_run); 3683 } 3684 3685 if (!backend_run) { 3686 virtio_set_status(vdev, vdev->status); 3687 } 3688 } 3689 3690 void virtio_instance_init_common(Object *proxy_obj, void *data, 3691 size_t vdev_size, const char *vdev_name) 3692 { 3693 DeviceState *vdev = data; 3694 3695 object_initialize_child_with_props(proxy_obj, "virtio-backend", vdev, 3696 vdev_size, vdev_name, &error_abort, 3697 NULL); 3698 qdev_alias_all_properties(vdev, proxy_obj); 3699 } 3700 3701 void virtio_init(VirtIODevice *vdev, uint16_t device_id, size_t config_size) 3702 { 3703 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3704 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3705 int i; 3706 int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0; 3707 3708 if (nvectors) { 3709 vdev->vector_queues = 3710 g_malloc0(sizeof(*vdev->vector_queues) * nvectors); 3711 } 3712 3713 vdev->start_on_kick = false; 3714 vdev->started = false; 3715 vdev->vhost_started = false; 3716 vdev->device_id = device_id; 3717 vdev->status = 0; 3718 qatomic_set(&vdev->isr, 0); 3719 vdev->queue_sel = 0; 3720 vdev->config_vector = VIRTIO_NO_VECTOR; 3721 vdev->vq = g_new0(VirtQueue, VIRTIO_QUEUE_MAX); 3722 vdev->vm_running = runstate_is_running(); 3723 vdev->broken = false; 3724 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 3725 vdev->vq[i].vector = VIRTIO_NO_VECTOR; 3726 vdev->vq[i].vdev = vdev; 3727 vdev->vq[i].queue_index = i; 3728 vdev->vq[i].host_notifier_enabled = false; 3729 } 3730 3731 vdev->name = virtio_id_to_name(device_id); 3732 vdev->config_len = config_size; 3733 if (vdev->config_len) { 3734 vdev->config = g_malloc0(config_size); 3735 } else { 3736 vdev->config = NULL; 3737 } 3738 vdev->vmstate = qdev_add_vm_change_state_handler(DEVICE(vdev), 3739 virtio_vmstate_change, vdev); 3740 vdev->device_endian = virtio_default_endian(); 3741 vdev->use_guest_notifier_mask = true; 3742 } 3743 3744 /* 3745 * Only devices that have already been around prior to defining the virtio 3746 * standard support legacy mode; this includes devices not specified in the 3747 * standard. All newer devices conform to the virtio standard only. 3748 */ 3749 bool virtio_legacy_allowed(VirtIODevice *vdev) 3750 { 3751 switch (vdev->device_id) { 3752 case VIRTIO_ID_NET: 3753 case VIRTIO_ID_BLOCK: 3754 case VIRTIO_ID_CONSOLE: 3755 case VIRTIO_ID_RNG: 3756 case VIRTIO_ID_BALLOON: 3757 case VIRTIO_ID_RPMSG: 3758 case VIRTIO_ID_SCSI: 3759 case VIRTIO_ID_9P: 3760 case VIRTIO_ID_RPROC_SERIAL: 3761 case VIRTIO_ID_CAIF: 3762 return true; 3763 default: 3764 return false; 3765 } 3766 } 3767 3768 bool virtio_legacy_check_disabled(VirtIODevice *vdev) 3769 { 3770 return vdev->disable_legacy_check; 3771 } 3772 3773 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n) 3774 { 3775 return vdev->vq[n].vring.desc; 3776 } 3777 3778 bool virtio_queue_enabled_legacy(VirtIODevice *vdev, int n) 3779 { 3780 return virtio_queue_get_desc_addr(vdev, n) != 0; 3781 } 3782 3783 bool virtio_queue_enabled(VirtIODevice *vdev, int n) 3784 { 3785 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3786 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3787 3788 if (k->queue_enabled) { 3789 return k->queue_enabled(qbus->parent, n); 3790 } 3791 return virtio_queue_enabled_legacy(vdev, n); 3792 } 3793 3794 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n) 3795 { 3796 return vdev->vq[n].vring.avail; 3797 } 3798 3799 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n) 3800 { 3801 return vdev->vq[n].vring.used; 3802 } 3803 3804 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n) 3805 { 3806 return sizeof(VRingDesc) * vdev->vq[n].vring.num; 3807 } 3808 3809 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n) 3810 { 3811 int s; 3812 3813 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3814 return sizeof(struct VRingPackedDescEvent); 3815 } 3816 3817 s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 3818 return offsetof(VRingAvail, ring) + 3819 sizeof(uint16_t) * vdev->vq[n].vring.num + s; 3820 } 3821 3822 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n) 3823 { 3824 int s; 3825 3826 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3827 return sizeof(struct VRingPackedDescEvent); 3828 } 3829 3830 s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 3831 return offsetof(VRingUsed, ring) + 3832 sizeof(VRingUsedElem) * vdev->vq[n].vring.num + s; 3833 } 3834 3835 static unsigned int virtio_queue_packed_get_last_avail_idx(VirtIODevice *vdev, 3836 int n) 3837 { 3838 unsigned int avail, used; 3839 3840 avail = vdev->vq[n].last_avail_idx; 3841 avail |= ((uint16_t)vdev->vq[n].last_avail_wrap_counter) << 15; 3842 3843 used = vdev->vq[n].used_idx; 3844 used |= ((uint16_t)vdev->vq[n].used_wrap_counter) << 15; 3845 3846 return avail | used << 16; 3847 } 3848 3849 static uint16_t virtio_queue_split_get_last_avail_idx(VirtIODevice *vdev, 3850 int n) 3851 { 3852 return vdev->vq[n].last_avail_idx; 3853 } 3854 3855 unsigned int virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n) 3856 { 3857 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3858 return virtio_queue_packed_get_last_avail_idx(vdev, n); 3859 } else { 3860 return virtio_queue_split_get_last_avail_idx(vdev, n); 3861 } 3862 } 3863 3864 static void virtio_queue_packed_set_last_avail_idx(VirtIODevice *vdev, 3865 int n, unsigned int idx) 3866 { 3867 struct VirtQueue *vq = &vdev->vq[n]; 3868 3869 vq->last_avail_idx = vq->shadow_avail_idx = idx & 0x7fff; 3870 vq->last_avail_wrap_counter = 3871 vq->shadow_avail_wrap_counter = !!(idx & 0x8000); 3872 idx >>= 16; 3873 vq->used_idx = idx & 0x7ffff; 3874 vq->used_wrap_counter = !!(idx & 0x8000); 3875 } 3876 3877 static void virtio_queue_split_set_last_avail_idx(VirtIODevice *vdev, 3878 int n, unsigned int idx) 3879 { 3880 vdev->vq[n].last_avail_idx = idx; 3881 vdev->vq[n].shadow_avail_idx = idx; 3882 } 3883 3884 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, 3885 unsigned int idx) 3886 { 3887 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3888 virtio_queue_packed_set_last_avail_idx(vdev, n, idx); 3889 } else { 3890 virtio_queue_split_set_last_avail_idx(vdev, n, idx); 3891 } 3892 } 3893 3894 static void virtio_queue_packed_restore_last_avail_idx(VirtIODevice *vdev, 3895 int n) 3896 { 3897 /* We don't have a reference like avail idx in shared memory */ 3898 return; 3899 } 3900 3901 static void virtio_queue_split_restore_last_avail_idx(VirtIODevice *vdev, 3902 int n) 3903 { 3904 RCU_READ_LOCK_GUARD(); 3905 if (vdev->vq[n].vring.desc) { 3906 vdev->vq[n].last_avail_idx = vring_used_idx(&vdev->vq[n]); 3907 vdev->vq[n].shadow_avail_idx = vdev->vq[n].last_avail_idx; 3908 } 3909 } 3910 3911 void virtio_queue_restore_last_avail_idx(VirtIODevice *vdev, int n) 3912 { 3913 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3914 virtio_queue_packed_restore_last_avail_idx(vdev, n); 3915 } else { 3916 virtio_queue_split_restore_last_avail_idx(vdev, n); 3917 } 3918 } 3919 3920 static void virtio_queue_packed_update_used_idx(VirtIODevice *vdev, int n) 3921 { 3922 /* used idx was updated through set_last_avail_idx() */ 3923 return; 3924 } 3925 3926 static void virtio_split_packed_update_used_idx(VirtIODevice *vdev, int n) 3927 { 3928 RCU_READ_LOCK_GUARD(); 3929 if (vdev->vq[n].vring.desc) { 3930 vdev->vq[n].used_idx = vring_used_idx(&vdev->vq[n]); 3931 } 3932 } 3933 3934 void virtio_queue_update_used_idx(VirtIODevice *vdev, int n) 3935 { 3936 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3937 return virtio_queue_packed_update_used_idx(vdev, n); 3938 } else { 3939 return virtio_split_packed_update_used_idx(vdev, n); 3940 } 3941 } 3942 3943 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n) 3944 { 3945 vdev->vq[n].signalled_used_valid = false; 3946 } 3947 3948 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n) 3949 { 3950 return vdev->vq + n; 3951 } 3952 3953 uint16_t virtio_get_queue_index(VirtQueue *vq) 3954 { 3955 return vq->queue_index; 3956 } 3957 3958 static void virtio_queue_guest_notifier_read(EventNotifier *n) 3959 { 3960 VirtQueue *vq = container_of(n, VirtQueue, guest_notifier); 3961 if (event_notifier_test_and_clear(n)) { 3962 virtio_irq(vq); 3963 } 3964 } 3965 3966 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign, 3967 bool with_irqfd) 3968 { 3969 if (assign && !with_irqfd) { 3970 event_notifier_set_handler(&vq->guest_notifier, 3971 virtio_queue_guest_notifier_read); 3972 } else { 3973 event_notifier_set_handler(&vq->guest_notifier, NULL); 3974 } 3975 if (!assign) { 3976 /* Test and clear notifier before closing it, 3977 * in case poll callback didn't have time to run. */ 3978 virtio_queue_guest_notifier_read(&vq->guest_notifier); 3979 } 3980 } 3981 3982 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq) 3983 { 3984 return &vq->guest_notifier; 3985 } 3986 3987 static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier *n) 3988 { 3989 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 3990 3991 virtio_queue_set_notification(vq, 0); 3992 } 3993 3994 static bool virtio_queue_host_notifier_aio_poll(void *opaque) 3995 { 3996 EventNotifier *n = opaque; 3997 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 3998 3999 return vq->vring.desc && !virtio_queue_empty(vq); 4000 } 4001 4002 static void virtio_queue_host_notifier_aio_poll_ready(EventNotifier *n) 4003 { 4004 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 4005 4006 virtio_queue_notify_vq(vq); 4007 } 4008 4009 static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n) 4010 { 4011 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 4012 4013 /* Caller polls once more after this to catch requests that race with us */ 4014 virtio_queue_set_notification(vq, 1); 4015 } 4016 4017 void virtio_queue_aio_attach_host_notifier(VirtQueue *vq, AioContext *ctx) 4018 { 4019 aio_set_event_notifier(ctx, &vq->host_notifier, true, 4020 virtio_queue_host_notifier_read, 4021 virtio_queue_host_notifier_aio_poll, 4022 virtio_queue_host_notifier_aio_poll_ready); 4023 aio_set_event_notifier_poll(ctx, &vq->host_notifier, 4024 virtio_queue_host_notifier_aio_poll_begin, 4025 virtio_queue_host_notifier_aio_poll_end); 4026 } 4027 4028 /* 4029 * Same as virtio_queue_aio_attach_host_notifier() but without polling. Use 4030 * this for rx virtqueues and similar cases where the virtqueue handler 4031 * function does not pop all elements. When the virtqueue is left non-empty 4032 * polling consumes CPU cycles and should not be used. 4033 */ 4034 void virtio_queue_aio_attach_host_notifier_no_poll(VirtQueue *vq, AioContext *ctx) 4035 { 4036 aio_set_event_notifier(ctx, &vq->host_notifier, true, 4037 virtio_queue_host_notifier_read, 4038 NULL, NULL); 4039 } 4040 4041 void virtio_queue_aio_detach_host_notifier(VirtQueue *vq, AioContext *ctx) 4042 { 4043 aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL, NULL, NULL); 4044 /* Test and clear notifier before after disabling event, 4045 * in case poll callback didn't have time to run. */ 4046 virtio_queue_host_notifier_read(&vq->host_notifier); 4047 } 4048 4049 void virtio_queue_host_notifier_read(EventNotifier *n) 4050 { 4051 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 4052 if (event_notifier_test_and_clear(n)) { 4053 virtio_queue_notify_vq(vq); 4054 } 4055 } 4056 4057 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq) 4058 { 4059 return &vq->host_notifier; 4060 } 4061 4062 void virtio_queue_set_host_notifier_enabled(VirtQueue *vq, bool enabled) 4063 { 4064 vq->host_notifier_enabled = enabled; 4065 } 4066 4067 int virtio_queue_set_host_notifier_mr(VirtIODevice *vdev, int n, 4068 MemoryRegion *mr, bool assign) 4069 { 4070 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 4071 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 4072 4073 if (k->set_host_notifier_mr) { 4074 return k->set_host_notifier_mr(qbus->parent, n, mr, assign); 4075 } 4076 4077 return -1; 4078 } 4079 4080 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name) 4081 { 4082 g_free(vdev->bus_name); 4083 vdev->bus_name = g_strdup(bus_name); 4084 } 4085 4086 void G_GNUC_PRINTF(2, 3) virtio_error(VirtIODevice *vdev, const char *fmt, ...) 4087 { 4088 va_list ap; 4089 4090 va_start(ap, fmt); 4091 error_vreport(fmt, ap); 4092 va_end(ap); 4093 4094 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 4095 vdev->status = vdev->status | VIRTIO_CONFIG_S_NEEDS_RESET; 4096 virtio_notify_config(vdev); 4097 } 4098 4099 vdev->broken = true; 4100 } 4101 4102 static void virtio_memory_listener_commit(MemoryListener *listener) 4103 { 4104 VirtIODevice *vdev = container_of(listener, VirtIODevice, listener); 4105 int i; 4106 4107 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 4108 if (vdev->vq[i].vring.num == 0) { 4109 break; 4110 } 4111 virtio_init_region_cache(vdev, i); 4112 } 4113 } 4114 4115 static void virtio_device_realize(DeviceState *dev, Error **errp) 4116 { 4117 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 4118 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 4119 Error *err = NULL; 4120 4121 /* Devices should either use vmsd or the load/save methods */ 4122 assert(!vdc->vmsd || !vdc->load); 4123 4124 if (vdc->realize != NULL) { 4125 vdc->realize(dev, &err); 4126 if (err != NULL) { 4127 error_propagate(errp, err); 4128 return; 4129 } 4130 } 4131 4132 virtio_bus_device_plugged(vdev, &err); 4133 if (err != NULL) { 4134 error_propagate(errp, err); 4135 vdc->unrealize(dev); 4136 return; 4137 } 4138 4139 vdev->listener.commit = virtio_memory_listener_commit; 4140 vdev->listener.name = "virtio"; 4141 memory_listener_register(&vdev->listener, vdev->dma_as); 4142 QTAILQ_INSERT_TAIL(&virtio_list, vdev, next); 4143 } 4144 4145 static void virtio_device_unrealize(DeviceState *dev) 4146 { 4147 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 4148 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 4149 4150 memory_listener_unregister(&vdev->listener); 4151 virtio_bus_device_unplugged(vdev); 4152 4153 if (vdc->unrealize != NULL) { 4154 vdc->unrealize(dev); 4155 } 4156 4157 QTAILQ_REMOVE(&virtio_list, vdev, next); 4158 g_free(vdev->bus_name); 4159 vdev->bus_name = NULL; 4160 } 4161 4162 static void virtio_device_free_virtqueues(VirtIODevice *vdev) 4163 { 4164 int i; 4165 if (!vdev->vq) { 4166 return; 4167 } 4168 4169 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 4170 if (vdev->vq[i].vring.num == 0) { 4171 break; 4172 } 4173 virtio_virtqueue_reset_region_cache(&vdev->vq[i]); 4174 } 4175 g_free(vdev->vq); 4176 } 4177 4178 static void virtio_device_instance_finalize(Object *obj) 4179 { 4180 VirtIODevice *vdev = VIRTIO_DEVICE(obj); 4181 4182 virtio_device_free_virtqueues(vdev); 4183 4184 g_free(vdev->config); 4185 g_free(vdev->vector_queues); 4186 } 4187 4188 static Property virtio_properties[] = { 4189 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features), 4190 DEFINE_PROP_BOOL("use-started", VirtIODevice, use_started, true), 4191 DEFINE_PROP_BOOL("use-disabled-flag", VirtIODevice, use_disabled_flag, true), 4192 DEFINE_PROP_BOOL("x-disable-legacy-check", VirtIODevice, 4193 disable_legacy_check, false), 4194 DEFINE_PROP_END_OF_LIST(), 4195 }; 4196 4197 static int virtio_device_start_ioeventfd_impl(VirtIODevice *vdev) 4198 { 4199 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev))); 4200 int i, n, r, err; 4201 4202 /* 4203 * Batch all the host notifiers in a single transaction to avoid 4204 * quadratic time complexity in address_space_update_ioeventfds(). 4205 */ 4206 memory_region_transaction_begin(); 4207 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 4208 VirtQueue *vq = &vdev->vq[n]; 4209 if (!virtio_queue_get_num(vdev, n)) { 4210 continue; 4211 } 4212 r = virtio_bus_set_host_notifier(qbus, n, true); 4213 if (r < 0) { 4214 err = r; 4215 goto assign_error; 4216 } 4217 event_notifier_set_handler(&vq->host_notifier, 4218 virtio_queue_host_notifier_read); 4219 } 4220 4221 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 4222 /* Kick right away to begin processing requests already in vring */ 4223 VirtQueue *vq = &vdev->vq[n]; 4224 if (!vq->vring.num) { 4225 continue; 4226 } 4227 event_notifier_set(&vq->host_notifier); 4228 } 4229 memory_region_transaction_commit(); 4230 return 0; 4231 4232 assign_error: 4233 i = n; /* save n for a second iteration after transaction is committed. */ 4234 while (--n >= 0) { 4235 VirtQueue *vq = &vdev->vq[n]; 4236 if (!virtio_queue_get_num(vdev, n)) { 4237 continue; 4238 } 4239 4240 event_notifier_set_handler(&vq->host_notifier, NULL); 4241 r = virtio_bus_set_host_notifier(qbus, n, false); 4242 assert(r >= 0); 4243 } 4244 /* 4245 * The transaction expects the ioeventfds to be open when it 4246 * commits. Do it now, before the cleanup loop. 4247 */ 4248 memory_region_transaction_commit(); 4249 4250 while (--i >= 0) { 4251 if (!virtio_queue_get_num(vdev, i)) { 4252 continue; 4253 } 4254 virtio_bus_cleanup_host_notifier(qbus, i); 4255 } 4256 return err; 4257 } 4258 4259 int virtio_device_start_ioeventfd(VirtIODevice *vdev) 4260 { 4261 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 4262 VirtioBusState *vbus = VIRTIO_BUS(qbus); 4263 4264 return virtio_bus_start_ioeventfd(vbus); 4265 } 4266 4267 static void virtio_device_stop_ioeventfd_impl(VirtIODevice *vdev) 4268 { 4269 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev))); 4270 int n, r; 4271 4272 /* 4273 * Batch all the host notifiers in a single transaction to avoid 4274 * quadratic time complexity in address_space_update_ioeventfds(). 4275 */ 4276 memory_region_transaction_begin(); 4277 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 4278 VirtQueue *vq = &vdev->vq[n]; 4279 4280 if (!virtio_queue_get_num(vdev, n)) { 4281 continue; 4282 } 4283 event_notifier_set_handler(&vq->host_notifier, NULL); 4284 r = virtio_bus_set_host_notifier(qbus, n, false); 4285 assert(r >= 0); 4286 } 4287 /* 4288 * The transaction expects the ioeventfds to be open when it 4289 * commits. Do it now, before the cleanup loop. 4290 */ 4291 memory_region_transaction_commit(); 4292 4293 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 4294 if (!virtio_queue_get_num(vdev, n)) { 4295 continue; 4296 } 4297 virtio_bus_cleanup_host_notifier(qbus, n); 4298 } 4299 } 4300 4301 int virtio_device_grab_ioeventfd(VirtIODevice *vdev) 4302 { 4303 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 4304 VirtioBusState *vbus = VIRTIO_BUS(qbus); 4305 4306 return virtio_bus_grab_ioeventfd(vbus); 4307 } 4308 4309 void virtio_device_release_ioeventfd(VirtIODevice *vdev) 4310 { 4311 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 4312 VirtioBusState *vbus = VIRTIO_BUS(qbus); 4313 4314 virtio_bus_release_ioeventfd(vbus); 4315 } 4316 4317 static void virtio_device_class_init(ObjectClass *klass, void *data) 4318 { 4319 /* Set the default value here. */ 4320 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 4321 DeviceClass *dc = DEVICE_CLASS(klass); 4322 4323 dc->realize = virtio_device_realize; 4324 dc->unrealize = virtio_device_unrealize; 4325 dc->bus_type = TYPE_VIRTIO_BUS; 4326 device_class_set_props(dc, virtio_properties); 4327 vdc->start_ioeventfd = virtio_device_start_ioeventfd_impl; 4328 vdc->stop_ioeventfd = virtio_device_stop_ioeventfd_impl; 4329 4330 vdc->legacy_features |= VIRTIO_LEGACY_FEATURES; 4331 4332 QTAILQ_INIT(&virtio_list); 4333 } 4334 4335 bool virtio_device_ioeventfd_enabled(VirtIODevice *vdev) 4336 { 4337 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 4338 VirtioBusState *vbus = VIRTIO_BUS(qbus); 4339 4340 return virtio_bus_ioeventfd_enabled(vbus); 4341 } 4342 4343 VirtioInfoList *qmp_x_query_virtio(Error **errp) 4344 { 4345 VirtioInfoList *list = NULL; 4346 VirtioInfoList *node; 4347 VirtIODevice *vdev; 4348 4349 QTAILQ_FOREACH(vdev, &virtio_list, next) { 4350 DeviceState *dev = DEVICE(vdev); 4351 Error *err = NULL; 4352 QObject *obj = qmp_qom_get(dev->canonical_path, "realized", &err); 4353 4354 if (err == NULL) { 4355 GString *is_realized = qobject_to_json_pretty(obj, true); 4356 /* virtio device is NOT realized, remove it from list */ 4357 if (!strncmp(is_realized->str, "false", 4)) { 4358 QTAILQ_REMOVE(&virtio_list, vdev, next); 4359 } else { 4360 node = g_new0(VirtioInfoList, 1); 4361 node->value = g_new(VirtioInfo, 1); 4362 node->value->path = g_strdup(dev->canonical_path); 4363 node->value->name = g_strdup(vdev->name); 4364 QAPI_LIST_PREPEND(list, node->value); 4365 } 4366 g_string_free(is_realized, true); 4367 } 4368 qobject_unref(obj); 4369 } 4370 4371 return list; 4372 } 4373 4374 static VirtIODevice *virtio_device_find(const char *path) 4375 { 4376 VirtIODevice *vdev; 4377 4378 QTAILQ_FOREACH(vdev, &virtio_list, next) { 4379 DeviceState *dev = DEVICE(vdev); 4380 4381 if (strcmp(dev->canonical_path, path) != 0) { 4382 continue; 4383 } 4384 4385 Error *err = NULL; 4386 QObject *obj = qmp_qom_get(dev->canonical_path, "realized", &err); 4387 if (err == NULL) { 4388 GString *is_realized = qobject_to_json_pretty(obj, true); 4389 /* virtio device is NOT realized, remove it from list */ 4390 if (!strncmp(is_realized->str, "false", 4)) { 4391 g_string_free(is_realized, true); 4392 qobject_unref(obj); 4393 QTAILQ_REMOVE(&virtio_list, vdev, next); 4394 return NULL; 4395 } 4396 g_string_free(is_realized, true); 4397 } else { 4398 /* virtio device doesn't exist in QOM tree */ 4399 QTAILQ_REMOVE(&virtio_list, vdev, next); 4400 qobject_unref(obj); 4401 return NULL; 4402 } 4403 /* device exists in QOM tree & is realized */ 4404 qobject_unref(obj); 4405 return vdev; 4406 } 4407 return NULL; 4408 } 4409 4410 #define CONVERT_FEATURES(type, map, is_status, bitmap) \ 4411 ({ \ 4412 type *list = NULL; \ 4413 type *node; \ 4414 for (i = 0; map[i].virtio_bit != -1; i++) { \ 4415 if (is_status) { \ 4416 bit = map[i].virtio_bit; \ 4417 } \ 4418 else { \ 4419 bit = 1ULL << map[i].virtio_bit; \ 4420 } \ 4421 if ((bitmap & bit) == 0) { \ 4422 continue; \ 4423 } \ 4424 node = g_new0(type, 1); \ 4425 node->value = g_strdup(map[i].feature_desc); \ 4426 node->next = list; \ 4427 list = node; \ 4428 bitmap ^= bit; \ 4429 } \ 4430 list; \ 4431 }) 4432 4433 static VirtioDeviceStatus *qmp_decode_status(uint8_t bitmap) 4434 { 4435 VirtioDeviceStatus *status; 4436 uint8_t bit; 4437 int i; 4438 4439 status = g_new0(VirtioDeviceStatus, 1); 4440 status->statuses = CONVERT_FEATURES(strList, virtio_config_status_map, 4441 1, bitmap); 4442 status->has_unknown_statuses = bitmap != 0; 4443 if (status->has_unknown_statuses) { 4444 status->unknown_statuses = bitmap; 4445 } 4446 4447 return status; 4448 } 4449 4450 static VhostDeviceProtocols *qmp_decode_protocols(uint64_t bitmap) 4451 { 4452 VhostDeviceProtocols *vhu_protocols; 4453 uint64_t bit; 4454 int i; 4455 4456 vhu_protocols = g_new0(VhostDeviceProtocols, 1); 4457 vhu_protocols->protocols = 4458 CONVERT_FEATURES(strList, 4459 vhost_user_protocol_map, 0, bitmap); 4460 vhu_protocols->has_unknown_protocols = bitmap != 0; 4461 if (vhu_protocols->has_unknown_protocols) { 4462 vhu_protocols->unknown_protocols = bitmap; 4463 } 4464 4465 return vhu_protocols; 4466 } 4467 4468 static VirtioDeviceFeatures *qmp_decode_features(uint16_t device_id, 4469 uint64_t bitmap) 4470 { 4471 VirtioDeviceFeatures *features; 4472 uint64_t bit; 4473 int i; 4474 4475 features = g_new0(VirtioDeviceFeatures, 1); 4476 features->has_dev_features = true; 4477 4478 /* transport features */ 4479 features->transports = CONVERT_FEATURES(strList, virtio_transport_map, 0, 4480 bitmap); 4481 4482 /* device features */ 4483 switch (device_id) { 4484 #ifdef CONFIG_VIRTIO_SERIAL 4485 case VIRTIO_ID_CONSOLE: 4486 features->dev_features = 4487 CONVERT_FEATURES(strList, virtio_serial_feature_map, 0, bitmap); 4488 break; 4489 #endif 4490 #ifdef CONFIG_VIRTIO_BLK 4491 case VIRTIO_ID_BLOCK: 4492 features->dev_features = 4493 CONVERT_FEATURES(strList, virtio_blk_feature_map, 0, bitmap); 4494 break; 4495 #endif 4496 #ifdef CONFIG_VIRTIO_GPU 4497 case VIRTIO_ID_GPU: 4498 features->dev_features = 4499 CONVERT_FEATURES(strList, virtio_gpu_feature_map, 0, bitmap); 4500 break; 4501 #endif 4502 #ifdef CONFIG_VIRTIO_NET 4503 case VIRTIO_ID_NET: 4504 features->dev_features = 4505 CONVERT_FEATURES(strList, virtio_net_feature_map, 0, bitmap); 4506 break; 4507 #endif 4508 #ifdef CONFIG_VIRTIO_SCSI 4509 case VIRTIO_ID_SCSI: 4510 features->dev_features = 4511 CONVERT_FEATURES(strList, virtio_scsi_feature_map, 0, bitmap); 4512 break; 4513 #endif 4514 #ifdef CONFIG_VIRTIO_BALLOON 4515 case VIRTIO_ID_BALLOON: 4516 features->dev_features = 4517 CONVERT_FEATURES(strList, virtio_balloon_feature_map, 0, bitmap); 4518 break; 4519 #endif 4520 #ifdef CONFIG_VIRTIO_IOMMU 4521 case VIRTIO_ID_IOMMU: 4522 features->dev_features = 4523 CONVERT_FEATURES(strList, virtio_iommu_feature_map, 0, bitmap); 4524 break; 4525 #endif 4526 #ifdef CONFIG_VIRTIO_INPUT 4527 case VIRTIO_ID_INPUT: 4528 features->dev_features = 4529 CONVERT_FEATURES(strList, virtio_input_feature_map, 0, bitmap); 4530 break; 4531 #endif 4532 #ifdef CONFIG_VHOST_USER_FS 4533 case VIRTIO_ID_FS: 4534 features->dev_features = 4535 CONVERT_FEATURES(strList, virtio_fs_feature_map, 0, bitmap); 4536 break; 4537 #endif 4538 #ifdef CONFIG_VHOST_VSOCK 4539 case VIRTIO_ID_VSOCK: 4540 features->dev_features = 4541 CONVERT_FEATURES(strList, virtio_vsock_feature_map, 0, bitmap); 4542 break; 4543 #endif 4544 #ifdef CONFIG_VIRTIO_CRYPTO 4545 case VIRTIO_ID_CRYPTO: 4546 features->dev_features = 4547 CONVERT_FEATURES(strList, virtio_crypto_feature_map, 0, bitmap); 4548 break; 4549 #endif 4550 #ifdef CONFIG_VIRTIO_MEM 4551 case VIRTIO_ID_MEM: 4552 features->dev_features = 4553 CONVERT_FEATURES(strList, virtio_mem_feature_map, 0, bitmap); 4554 break; 4555 #endif 4556 #ifdef CONFIG_VIRTIO_I2C_ADAPTER 4557 case VIRTIO_ID_I2C_ADAPTER: 4558 features->dev_features = 4559 CONVERT_FEATURES(strList, virtio_i2c_feature_map, 0, bitmap); 4560 break; 4561 #endif 4562 #ifdef CONFIG_VIRTIO_RNG 4563 case VIRTIO_ID_RNG: 4564 features->dev_features = 4565 CONVERT_FEATURES(strList, virtio_rng_feature_map, 0, bitmap); 4566 break; 4567 #endif 4568 /* No features */ 4569 case VIRTIO_ID_9P: 4570 case VIRTIO_ID_PMEM: 4571 case VIRTIO_ID_IOMEM: 4572 case VIRTIO_ID_RPMSG: 4573 case VIRTIO_ID_CLOCK: 4574 case VIRTIO_ID_MAC80211_WLAN: 4575 case VIRTIO_ID_MAC80211_HWSIM: 4576 case VIRTIO_ID_RPROC_SERIAL: 4577 case VIRTIO_ID_MEMORY_BALLOON: 4578 case VIRTIO_ID_CAIF: 4579 case VIRTIO_ID_SIGNAL_DIST: 4580 case VIRTIO_ID_PSTORE: 4581 case VIRTIO_ID_SOUND: 4582 case VIRTIO_ID_BT: 4583 case VIRTIO_ID_RPMB: 4584 case VIRTIO_ID_VIDEO_ENCODER: 4585 case VIRTIO_ID_VIDEO_DECODER: 4586 case VIRTIO_ID_SCMI: 4587 case VIRTIO_ID_NITRO_SEC_MOD: 4588 case VIRTIO_ID_WATCHDOG: 4589 case VIRTIO_ID_CAN: 4590 case VIRTIO_ID_DMABUF: 4591 case VIRTIO_ID_PARAM_SERV: 4592 case VIRTIO_ID_AUDIO_POLICY: 4593 case VIRTIO_ID_GPIO: 4594 break; 4595 default: 4596 g_assert_not_reached(); 4597 } 4598 4599 features->has_unknown_dev_features = bitmap != 0; 4600 if (features->has_unknown_dev_features) { 4601 features->unknown_dev_features = bitmap; 4602 } 4603 4604 return features; 4605 } 4606 4607 VirtioStatus *qmp_x_query_virtio_status(const char *path, Error **errp) 4608 { 4609 VirtIODevice *vdev; 4610 VirtioStatus *status; 4611 4612 vdev = virtio_device_find(path); 4613 if (vdev == NULL) { 4614 error_setg(errp, "Path %s is not a VirtIODevice", path); 4615 return NULL; 4616 } 4617 4618 status = g_new0(VirtioStatus, 1); 4619 status->name = g_strdup(vdev->name); 4620 status->device_id = vdev->device_id; 4621 status->vhost_started = vdev->vhost_started; 4622 status->guest_features = qmp_decode_features(vdev->device_id, 4623 vdev->guest_features); 4624 status->host_features = qmp_decode_features(vdev->device_id, 4625 vdev->host_features); 4626 status->backend_features = qmp_decode_features(vdev->device_id, 4627 vdev->backend_features); 4628 4629 switch (vdev->device_endian) { 4630 case VIRTIO_DEVICE_ENDIAN_LITTLE: 4631 status->device_endian = g_strdup("little"); 4632 break; 4633 case VIRTIO_DEVICE_ENDIAN_BIG: 4634 status->device_endian = g_strdup("big"); 4635 break; 4636 default: 4637 status->device_endian = g_strdup("unknown"); 4638 break; 4639 } 4640 4641 status->num_vqs = virtio_get_num_queues(vdev); 4642 status->status = qmp_decode_status(vdev->status); 4643 status->isr = vdev->isr; 4644 status->queue_sel = vdev->queue_sel; 4645 status->vm_running = vdev->vm_running; 4646 status->broken = vdev->broken; 4647 status->disabled = vdev->disabled; 4648 status->use_started = vdev->use_started; 4649 status->started = vdev->started; 4650 status->start_on_kick = vdev->start_on_kick; 4651 status->disable_legacy_check = vdev->disable_legacy_check; 4652 status->bus_name = g_strdup(vdev->bus_name); 4653 status->use_guest_notifier_mask = vdev->use_guest_notifier_mask; 4654 status->has_vhost_dev = vdev->vhost_started; 4655 4656 if (vdev->vhost_started) { 4657 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 4658 struct vhost_dev *hdev = vdc->get_vhost(vdev); 4659 4660 status->vhost_dev = g_new0(VhostStatus, 1); 4661 status->vhost_dev->n_mem_sections = hdev->n_mem_sections; 4662 status->vhost_dev->n_tmp_sections = hdev->n_tmp_sections; 4663 status->vhost_dev->nvqs = hdev->nvqs; 4664 status->vhost_dev->vq_index = hdev->vq_index; 4665 status->vhost_dev->features = 4666 qmp_decode_features(vdev->device_id, hdev->features); 4667 status->vhost_dev->acked_features = 4668 qmp_decode_features(vdev->device_id, hdev->acked_features); 4669 status->vhost_dev->backend_features = 4670 qmp_decode_features(vdev->device_id, hdev->backend_features); 4671 status->vhost_dev->protocol_features = 4672 qmp_decode_protocols(hdev->protocol_features); 4673 status->vhost_dev->max_queues = hdev->max_queues; 4674 status->vhost_dev->backend_cap = hdev->backend_cap; 4675 status->vhost_dev->log_enabled = hdev->log_enabled; 4676 status->vhost_dev->log_size = hdev->log_size; 4677 } 4678 4679 return status; 4680 } 4681 4682 static const TypeInfo virtio_device_info = { 4683 .name = TYPE_VIRTIO_DEVICE, 4684 .parent = TYPE_DEVICE, 4685 .instance_size = sizeof(VirtIODevice), 4686 .class_init = virtio_device_class_init, 4687 .instance_finalize = virtio_device_instance_finalize, 4688 .abstract = true, 4689 .class_size = sizeof(VirtioDeviceClass), 4690 }; 4691 4692 static void virtio_register_types(void) 4693 { 4694 type_register_static(&virtio_device_info); 4695 } 4696 4697 type_init(virtio_register_types) 4698