1 /* 2 * Virtio Support 3 * 4 * Copyright IBM, Corp. 2007 5 * 6 * Authors: 7 * Anthony Liguori <aliguori@us.ibm.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 */ 13 14 #include "qemu/osdep.h" 15 #include "qapi/error.h" 16 #include "qapi/qmp/qdict.h" 17 #include "qapi/qapi-commands-virtio.h" 18 #include "qapi/qapi-commands-qom.h" 19 #include "qapi/qapi-visit-virtio.h" 20 #include "qapi/qmp/qjson.h" 21 #include "cpu.h" 22 #include "trace.h" 23 #include "qemu/error-report.h" 24 #include "qemu/log.h" 25 #include "qemu/main-loop.h" 26 #include "qemu/module.h" 27 #include "qom/object_interfaces.h" 28 #include "hw/virtio/virtio.h" 29 #include "migration/qemu-file-types.h" 30 #include "qemu/atomic.h" 31 #include "hw/virtio/virtio-bus.h" 32 #include "hw/qdev-properties.h" 33 #include "hw/virtio/virtio-access.h" 34 #include "sysemu/dma.h" 35 #include "sysemu/runstate.h" 36 #include "standard-headers/linux/virtio_ids.h" 37 #include "standard-headers/linux/vhost_types.h" 38 #include "standard-headers/linux/virtio_blk.h" 39 #include "standard-headers/linux/virtio_console.h" 40 #include "standard-headers/linux/virtio_gpu.h" 41 #include "standard-headers/linux/virtio_net.h" 42 #include "standard-headers/linux/virtio_scsi.h" 43 #include "standard-headers/linux/virtio_i2c.h" 44 #include "standard-headers/linux/virtio_balloon.h" 45 #include "standard-headers/linux/virtio_iommu.h" 46 #include "standard-headers/linux/virtio_mem.h" 47 #include "standard-headers/linux/virtio_vsock.h" 48 #include CONFIG_DEVICES 49 50 /* QAPI list of realized VirtIODevices */ 51 static QTAILQ_HEAD(, VirtIODevice) virtio_list; 52 53 /* 54 * Maximum size of virtio device config space 55 */ 56 #define VHOST_USER_MAX_CONFIG_SIZE 256 57 58 #define FEATURE_ENTRY(name, desc) (qmp_virtio_feature_map_t) \ 59 { .virtio_bit = name, .feature_desc = desc } 60 61 enum VhostUserProtocolFeature { 62 VHOST_USER_PROTOCOL_F_MQ = 0, 63 VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1, 64 VHOST_USER_PROTOCOL_F_RARP = 2, 65 VHOST_USER_PROTOCOL_F_REPLY_ACK = 3, 66 VHOST_USER_PROTOCOL_F_NET_MTU = 4, 67 VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5, 68 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6, 69 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7, 70 VHOST_USER_PROTOCOL_F_PAGEFAULT = 8, 71 VHOST_USER_PROTOCOL_F_CONFIG = 9, 72 VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10, 73 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11, 74 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD = 12, 75 VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13, 76 VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS = 14, 77 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS = 15, 78 VHOST_USER_PROTOCOL_F_MAX 79 }; 80 81 /* Virtio transport features mapping */ 82 static qmp_virtio_feature_map_t virtio_transport_map[] = { 83 /* Virtio device transport features */ 84 #ifndef VIRTIO_CONFIG_NO_LEGACY 85 FEATURE_ENTRY(VIRTIO_F_NOTIFY_ON_EMPTY, \ 86 "VIRTIO_F_NOTIFY_ON_EMPTY: Notify when device runs out of avail. " 87 "descs. on VQ"), 88 FEATURE_ENTRY(VIRTIO_F_ANY_LAYOUT, \ 89 "VIRTIO_F_ANY_LAYOUT: Device accepts arbitrary desc. layouts"), 90 #endif /* !VIRTIO_CONFIG_NO_LEGACY */ 91 FEATURE_ENTRY(VIRTIO_F_VERSION_1, \ 92 "VIRTIO_F_VERSION_1: Device compliant for v1 spec (legacy)"), 93 FEATURE_ENTRY(VIRTIO_F_IOMMU_PLATFORM, \ 94 "VIRTIO_F_IOMMU_PLATFORM: Device can be used on IOMMU platform"), 95 FEATURE_ENTRY(VIRTIO_F_RING_PACKED, \ 96 "VIRTIO_F_RING_PACKED: Device supports packed VQ layout"), 97 FEATURE_ENTRY(VIRTIO_F_IN_ORDER, \ 98 "VIRTIO_F_IN_ORDER: Device uses buffers in same order as made " 99 "available by driver"), 100 FEATURE_ENTRY(VIRTIO_F_ORDER_PLATFORM, \ 101 "VIRTIO_F_ORDER_PLATFORM: Memory accesses ordered by platform"), 102 FEATURE_ENTRY(VIRTIO_F_SR_IOV, \ 103 "VIRTIO_F_SR_IOV: Device supports single root I/O virtualization"), 104 /* Virtio ring transport features */ 105 FEATURE_ENTRY(VIRTIO_RING_F_INDIRECT_DESC, \ 106 "VIRTIO_RING_F_INDIRECT_DESC: Indirect descriptors supported"), 107 FEATURE_ENTRY(VIRTIO_RING_F_EVENT_IDX, \ 108 "VIRTIO_RING_F_EVENT_IDX: Used & avail. event fields enabled"), 109 { -1, "" } 110 }; 111 112 /* Vhost-user protocol features mapping */ 113 static qmp_virtio_feature_map_t vhost_user_protocol_map[] = { 114 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_MQ, \ 115 "VHOST_USER_PROTOCOL_F_MQ: Multiqueue protocol supported"), 116 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_LOG_SHMFD, \ 117 "VHOST_USER_PROTOCOL_F_LOG_SHMFD: Shared log memory fd supported"), 118 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_RARP, \ 119 "VHOST_USER_PROTOCOL_F_RARP: Vhost-user back-end RARP broadcasting " 120 "supported"), 121 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_REPLY_ACK, \ 122 "VHOST_USER_PROTOCOL_F_REPLY_ACK: Requested operation status ack. " 123 "supported"), 124 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_NET_MTU, \ 125 "VHOST_USER_PROTOCOL_F_NET_MTU: Expose host MTU to guest supported"), 126 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_SLAVE_REQ, \ 127 "VHOST_USER_PROTOCOL_F_SLAVE_REQ: Socket fd for back-end initiated " 128 "requests supported"), 129 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_CROSS_ENDIAN, \ 130 "VHOST_USER_PROTOCOL_F_CROSS_ENDIAN: Endianness of VQs for legacy " 131 "devices supported"), 132 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_CRYPTO_SESSION, \ 133 "VHOST_USER_PROTOCOL_F_CRYPTO_SESSION: Session creation for crypto " 134 "operations supported"), 135 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_PAGEFAULT, \ 136 "VHOST_USER_PROTOCOL_F_PAGEFAULT: Request servicing on userfaultfd " 137 "for accessed pages supported"), 138 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_CONFIG, \ 139 "VHOST_USER_PROTOCOL_F_CONFIG: Vhost-user messaging for virtio " 140 "device configuration space supported"), 141 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD, \ 142 "VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD: Slave fd communication " 143 "channel supported"), 144 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_HOST_NOTIFIER, \ 145 "VHOST_USER_PROTOCOL_F_HOST_NOTIFIER: Host notifiers for specified " 146 "VQs supported"), 147 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD, \ 148 "VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD: Shared inflight I/O buffers " 149 "supported"), 150 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_RESET_DEVICE, \ 151 "VHOST_USER_PROTOCOL_F_RESET_DEVICE: Disabling all rings and " 152 "resetting internal device state supported"), 153 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS, \ 154 "VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS: In-band messaging " 155 "supported"), 156 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS, \ 157 "VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS: Configuration for " 158 "memory slots supported"), 159 { -1, "" } 160 }; 161 162 /* virtio device configuration statuses */ 163 static qmp_virtio_feature_map_t virtio_config_status_map[] = { 164 FEATURE_ENTRY(VIRTIO_CONFIG_S_DRIVER_OK, \ 165 "VIRTIO_CONFIG_S_DRIVER_OK: Driver setup and ready"), 166 FEATURE_ENTRY(VIRTIO_CONFIG_S_FEATURES_OK, \ 167 "VIRTIO_CONFIG_S_FEATURES_OK: Feature negotiation complete"), 168 FEATURE_ENTRY(VIRTIO_CONFIG_S_DRIVER, \ 169 "VIRTIO_CONFIG_S_DRIVER: Guest OS compatible with device"), 170 FEATURE_ENTRY(VIRTIO_CONFIG_S_NEEDS_RESET, \ 171 "VIRTIO_CONFIG_S_NEEDS_RESET: Irrecoverable error, device needs " 172 "reset"), 173 FEATURE_ENTRY(VIRTIO_CONFIG_S_FAILED, \ 174 "VIRTIO_CONFIG_S_FAILED: Error in guest, device failed"), 175 FEATURE_ENTRY(VIRTIO_CONFIG_S_ACKNOWLEDGE, \ 176 "VIRTIO_CONFIG_S_ACKNOWLEDGE: Valid virtio device found"), 177 { -1, "" } 178 }; 179 180 /* virtio-blk features mapping */ 181 qmp_virtio_feature_map_t virtio_blk_feature_map[] = { 182 FEATURE_ENTRY(VIRTIO_BLK_F_SIZE_MAX, \ 183 "VIRTIO_BLK_F_SIZE_MAX: Max segment size is size_max"), 184 FEATURE_ENTRY(VIRTIO_BLK_F_SEG_MAX, \ 185 "VIRTIO_BLK_F_SEG_MAX: Max segments in a request is seg_max"), 186 FEATURE_ENTRY(VIRTIO_BLK_F_GEOMETRY, \ 187 "VIRTIO_BLK_F_GEOMETRY: Legacy geometry available"), 188 FEATURE_ENTRY(VIRTIO_BLK_F_RO, \ 189 "VIRTIO_BLK_F_RO: Device is read-only"), 190 FEATURE_ENTRY(VIRTIO_BLK_F_BLK_SIZE, \ 191 "VIRTIO_BLK_F_BLK_SIZE: Block size of disk available"), 192 FEATURE_ENTRY(VIRTIO_BLK_F_TOPOLOGY, \ 193 "VIRTIO_BLK_F_TOPOLOGY: Topology information available"), 194 FEATURE_ENTRY(VIRTIO_BLK_F_MQ, \ 195 "VIRTIO_BLK_F_MQ: Multiqueue supported"), 196 FEATURE_ENTRY(VIRTIO_BLK_F_DISCARD, \ 197 "VIRTIO_BLK_F_DISCARD: Discard command supported"), 198 FEATURE_ENTRY(VIRTIO_BLK_F_WRITE_ZEROES, \ 199 "VIRTIO_BLK_F_WRITE_ZEROES: Write zeroes command supported"), 200 #ifndef VIRTIO_BLK_NO_LEGACY 201 FEATURE_ENTRY(VIRTIO_BLK_F_BARRIER, \ 202 "VIRTIO_BLK_F_BARRIER: Request barriers supported"), 203 FEATURE_ENTRY(VIRTIO_BLK_F_SCSI, \ 204 "VIRTIO_BLK_F_SCSI: SCSI packet commands supported"), 205 FEATURE_ENTRY(VIRTIO_BLK_F_FLUSH, \ 206 "VIRTIO_BLK_F_FLUSH: Flush command supported"), 207 FEATURE_ENTRY(VIRTIO_BLK_F_CONFIG_WCE, \ 208 "VIRTIO_BLK_F_CONFIG_WCE: Cache writeback and writethrough modes " 209 "supported"), 210 #endif /* !VIRTIO_BLK_NO_LEGACY */ 211 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 212 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 213 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 214 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 215 "negotiation supported"), 216 { -1, "" } 217 }; 218 219 /* virtio-serial features mapping */ 220 qmp_virtio_feature_map_t virtio_serial_feature_map[] = { 221 FEATURE_ENTRY(VIRTIO_CONSOLE_F_SIZE, \ 222 "VIRTIO_CONSOLE_F_SIZE: Host providing console size"), 223 FEATURE_ENTRY(VIRTIO_CONSOLE_F_MULTIPORT, \ 224 "VIRTIO_CONSOLE_F_MULTIPORT: Multiple ports for device supported"), 225 FEATURE_ENTRY(VIRTIO_CONSOLE_F_EMERG_WRITE, \ 226 "VIRTIO_CONSOLE_F_EMERG_WRITE: Emergency write supported"), 227 { -1, "" } 228 }; 229 230 /* virtio-gpu features mapping */ 231 qmp_virtio_feature_map_t virtio_gpu_feature_map[] = { 232 FEATURE_ENTRY(VIRTIO_GPU_F_VIRGL, \ 233 "VIRTIO_GPU_F_VIRGL: Virgl 3D mode supported"), 234 FEATURE_ENTRY(VIRTIO_GPU_F_EDID, \ 235 "VIRTIO_GPU_F_EDID: EDID metadata supported"), 236 FEATURE_ENTRY(VIRTIO_GPU_F_RESOURCE_UUID, \ 237 "VIRTIO_GPU_F_RESOURCE_UUID: Resource UUID assigning supported"), 238 FEATURE_ENTRY(VIRTIO_GPU_F_RESOURCE_BLOB, \ 239 "VIRTIO_GPU_F_RESOURCE_BLOB: Size-based blob resources supported"), 240 FEATURE_ENTRY(VIRTIO_GPU_F_CONTEXT_INIT, \ 241 "VIRTIO_GPU_F_CONTEXT_INIT: Context types and synchronization " 242 "timelines supported"), 243 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 244 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 245 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 246 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 247 "negotiation supported"), 248 { -1, "" } 249 }; 250 251 /* virtio-input features mapping */ 252 qmp_virtio_feature_map_t virtio_input_feature_map[] = { 253 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 254 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 255 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 256 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 257 "negotiation supported"), 258 { -1, "" } 259 }; 260 261 /* virtio-net features mapping */ 262 qmp_virtio_feature_map_t virtio_net_feature_map[] = { 263 FEATURE_ENTRY(VIRTIO_NET_F_CSUM, \ 264 "VIRTIO_NET_F_CSUM: Device handling packets with partial checksum " 265 "supported"), 266 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_CSUM, \ 267 "VIRTIO_NET_F_GUEST_CSUM: Driver handling packets with partial " 268 "checksum supported"), 269 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \ 270 "VIRTIO_NET_F_CTRL_GUEST_OFFLOADS: Control channel offloading " 271 "reconfig. supported"), 272 FEATURE_ENTRY(VIRTIO_NET_F_MTU, \ 273 "VIRTIO_NET_F_MTU: Device max MTU reporting supported"), 274 FEATURE_ENTRY(VIRTIO_NET_F_MAC, \ 275 "VIRTIO_NET_F_MAC: Device has given MAC address"), 276 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_TSO4, \ 277 "VIRTIO_NET_F_GUEST_TSO4: Driver can receive TSOv4"), 278 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_TSO6, \ 279 "VIRTIO_NET_F_GUEST_TSO6: Driver can receive TSOv6"), 280 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_ECN, \ 281 "VIRTIO_NET_F_GUEST_ECN: Driver can receive TSO with ECN"), 282 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_UFO, \ 283 "VIRTIO_NET_F_GUEST_UFO: Driver can receive UFO"), 284 FEATURE_ENTRY(VIRTIO_NET_F_HOST_TSO4, \ 285 "VIRTIO_NET_F_HOST_TSO4: Device can receive TSOv4"), 286 FEATURE_ENTRY(VIRTIO_NET_F_HOST_TSO6, \ 287 "VIRTIO_NET_F_HOST_TSO6: Device can receive TSOv6"), 288 FEATURE_ENTRY(VIRTIO_NET_F_HOST_ECN, \ 289 "VIRTIO_NET_F_HOST_ECN: Device can receive TSO with ECN"), 290 FEATURE_ENTRY(VIRTIO_NET_F_HOST_UFO, \ 291 "VIRTIO_NET_F_HOST_UFO: Device can receive UFO"), 292 FEATURE_ENTRY(VIRTIO_NET_F_MRG_RXBUF, \ 293 "VIRTIO_NET_F_MRG_RXBUF: Driver can merge receive buffers"), 294 FEATURE_ENTRY(VIRTIO_NET_F_STATUS, \ 295 "VIRTIO_NET_F_STATUS: Configuration status field available"), 296 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_VQ, \ 297 "VIRTIO_NET_F_CTRL_VQ: Control channel available"), 298 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_RX, \ 299 "VIRTIO_NET_F_CTRL_RX: Control channel RX mode supported"), 300 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_VLAN, \ 301 "VIRTIO_NET_F_CTRL_VLAN: Control channel VLAN filtering supported"), 302 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_RX_EXTRA, \ 303 "VIRTIO_NET_F_CTRL_RX_EXTRA: Extra RX mode control supported"), 304 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_ANNOUNCE, \ 305 "VIRTIO_NET_F_GUEST_ANNOUNCE: Driver sending gratuitous packets " 306 "supported"), 307 FEATURE_ENTRY(VIRTIO_NET_F_MQ, \ 308 "VIRTIO_NET_F_MQ: Multiqueue with automatic receive steering " 309 "supported"), 310 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_MAC_ADDR, \ 311 "VIRTIO_NET_F_CTRL_MAC_ADDR: MAC address set through control " 312 "channel"), 313 FEATURE_ENTRY(VIRTIO_NET_F_HASH_REPORT, \ 314 "VIRTIO_NET_F_HASH_REPORT: Hash reporting supported"), 315 FEATURE_ENTRY(VIRTIO_NET_F_RSS, \ 316 "VIRTIO_NET_F_RSS: RSS RX steering supported"), 317 FEATURE_ENTRY(VIRTIO_NET_F_RSC_EXT, \ 318 "VIRTIO_NET_F_RSC_EXT: Extended coalescing info supported"), 319 FEATURE_ENTRY(VIRTIO_NET_F_STANDBY, \ 320 "VIRTIO_NET_F_STANDBY: Device acting as standby for primary " 321 "device with same MAC addr. supported"), 322 FEATURE_ENTRY(VIRTIO_NET_F_SPEED_DUPLEX, \ 323 "VIRTIO_NET_F_SPEED_DUPLEX: Device set linkspeed and duplex"), 324 #ifndef VIRTIO_NET_NO_LEGACY 325 FEATURE_ENTRY(VIRTIO_NET_F_GSO, \ 326 "VIRTIO_NET_F_GSO: Handling GSO-type packets supported"), 327 #endif /* !VIRTIO_NET_NO_LEGACY */ 328 FEATURE_ENTRY(VHOST_NET_F_VIRTIO_NET_HDR, \ 329 "VHOST_NET_F_VIRTIO_NET_HDR: Virtio-net headers for RX and TX " 330 "packets supported"), 331 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 332 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 333 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 334 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 335 "negotiation supported"), 336 { -1, "" } 337 }; 338 339 /* virtio-scsi features mapping */ 340 qmp_virtio_feature_map_t virtio_scsi_feature_map[] = { 341 FEATURE_ENTRY(VIRTIO_SCSI_F_INOUT, \ 342 "VIRTIO_SCSI_F_INOUT: Requests including read and writable data " 343 "buffers suppoted"), 344 FEATURE_ENTRY(VIRTIO_SCSI_F_HOTPLUG, \ 345 "VIRTIO_SCSI_F_HOTPLUG: Reporting and handling hot-plug events " 346 "supported"), 347 FEATURE_ENTRY(VIRTIO_SCSI_F_CHANGE, \ 348 "VIRTIO_SCSI_F_CHANGE: Reporting and handling LUN changes " 349 "supported"), 350 FEATURE_ENTRY(VIRTIO_SCSI_F_T10_PI, \ 351 "VIRTIO_SCSI_F_T10_PI: T10 info included in request header"), 352 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 353 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 354 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 355 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 356 "negotiation supported"), 357 { -1, "" } 358 }; 359 360 /* virtio/vhost-user-fs features mapping */ 361 qmp_virtio_feature_map_t virtio_fs_feature_map[] = { 362 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 363 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 364 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 365 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 366 "negotiation supported"), 367 { -1, "" } 368 }; 369 370 /* virtio/vhost-user-i2c features mapping */ 371 qmp_virtio_feature_map_t virtio_i2c_feature_map[] = { 372 FEATURE_ENTRY(VIRTIO_I2C_F_ZERO_LENGTH_REQUEST, \ 373 "VIRTIO_I2C_F_ZERO_LEGNTH_REQUEST: Zero length requests supported"), 374 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 375 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 376 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 377 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 378 "negotiation supported"), 379 { -1, "" } 380 }; 381 382 /* virtio/vhost-vsock features mapping */ 383 qmp_virtio_feature_map_t virtio_vsock_feature_map[] = { 384 FEATURE_ENTRY(VIRTIO_VSOCK_F_SEQPACKET, \ 385 "VIRTIO_VSOCK_F_SEQPACKET: SOCK_SEQPACKET supported"), 386 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 387 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 388 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 389 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 390 "negotiation supported"), 391 { -1, "" } 392 }; 393 394 /* virtio-balloon features mapping */ 395 qmp_virtio_feature_map_t virtio_balloon_feature_map[] = { 396 FEATURE_ENTRY(VIRTIO_BALLOON_F_MUST_TELL_HOST, \ 397 "VIRTIO_BALLOON_F_MUST_TELL_HOST: Tell host before reclaiming " 398 "pages"), 399 FEATURE_ENTRY(VIRTIO_BALLOON_F_STATS_VQ, \ 400 "VIRTIO_BALLOON_F_STATS_VQ: Guest memory stats VQ available"), 401 FEATURE_ENTRY(VIRTIO_BALLOON_F_DEFLATE_ON_OOM, \ 402 "VIRTIO_BALLOON_F_DEFLATE_ON_OOM: Deflate balloon when guest OOM"), 403 FEATURE_ENTRY(VIRTIO_BALLOON_F_FREE_PAGE_HINT, \ 404 "VIRTIO_BALLOON_F_FREE_PAGE_HINT: VQ reporting free pages enabled"), 405 FEATURE_ENTRY(VIRTIO_BALLOON_F_PAGE_POISON, \ 406 "VIRTIO_BALLOON_F_PAGE_POISON: Guest page poisoning enabled"), 407 FEATURE_ENTRY(VIRTIO_BALLOON_F_REPORTING, \ 408 "VIRTIO_BALLOON_F_REPORTING: Page reporting VQ enabled"), 409 { -1, "" } 410 }; 411 412 /* virtio-crypto features mapping */ 413 qmp_virtio_feature_map_t virtio_crypto_feature_map[] = { 414 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 415 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 416 { -1, "" } 417 }; 418 419 /* virtio-iommu features mapping */ 420 qmp_virtio_feature_map_t virtio_iommu_feature_map[] = { 421 FEATURE_ENTRY(VIRTIO_IOMMU_F_INPUT_RANGE, \ 422 "VIRTIO_IOMMU_F_INPUT_RANGE: Range of available virtual addrs. " 423 "available"), 424 FEATURE_ENTRY(VIRTIO_IOMMU_F_DOMAIN_RANGE, \ 425 "VIRTIO_IOMMU_F_DOMAIN_RANGE: Number of supported domains " 426 "available"), 427 FEATURE_ENTRY(VIRTIO_IOMMU_F_MAP_UNMAP, \ 428 "VIRTIO_IOMMU_F_MAP_UNMAP: Map and unmap requests available"), 429 FEATURE_ENTRY(VIRTIO_IOMMU_F_BYPASS, \ 430 "VIRTIO_IOMMU_F_BYPASS: Endpoints not attached to domains are in " 431 "bypass mode"), 432 FEATURE_ENTRY(VIRTIO_IOMMU_F_PROBE, \ 433 "VIRTIO_IOMMU_F_PROBE: Probe requests available"), 434 FEATURE_ENTRY(VIRTIO_IOMMU_F_MMIO, \ 435 "VIRTIO_IOMMU_F_MMIO: VIRTIO_IOMMU_MAP_F_MMIO flag available"), 436 FEATURE_ENTRY(VIRTIO_IOMMU_F_BYPASS_CONFIG, \ 437 "VIRTIO_IOMMU_F_BYPASS_CONFIG: Bypass field of IOMMU config " 438 "available"), 439 { -1, "" } 440 }; 441 442 /* virtio-mem features mapping */ 443 qmp_virtio_feature_map_t virtio_mem_feature_map[] = { 444 #ifndef CONFIG_ACPI 445 FEATURE_ENTRY(VIRTIO_MEM_F_ACPI_PXM, \ 446 "VIRTIO_MEM_F_ACPI_PXM: node_id is an ACPI PXM and is valid"), 447 #endif /* !CONFIG_ACPI */ 448 FEATURE_ENTRY(VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE, \ 449 "VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE: Unplugged memory cannot be " 450 "accessed"), 451 { -1, "" } 452 }; 453 454 /* virtio-rng features mapping */ 455 qmp_virtio_feature_map_t virtio_rng_feature_map[] = { 456 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 457 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 458 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 459 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 460 "negotiation supported"), 461 { -1, "" } 462 }; 463 464 /* 465 * The alignment to use between consumer and producer parts of vring. 466 * x86 pagesize again. This is the default, used by transports like PCI 467 * which don't provide a means for the guest to tell the host the alignment. 468 */ 469 #define VIRTIO_PCI_VRING_ALIGN 4096 470 471 typedef struct VRingDesc 472 { 473 uint64_t addr; 474 uint32_t len; 475 uint16_t flags; 476 uint16_t next; 477 } VRingDesc; 478 479 typedef struct VRingPackedDesc { 480 uint64_t addr; 481 uint32_t len; 482 uint16_t id; 483 uint16_t flags; 484 } VRingPackedDesc; 485 486 typedef struct VRingAvail 487 { 488 uint16_t flags; 489 uint16_t idx; 490 uint16_t ring[]; 491 } VRingAvail; 492 493 typedef struct VRingUsedElem 494 { 495 uint32_t id; 496 uint32_t len; 497 } VRingUsedElem; 498 499 typedef struct VRingUsed 500 { 501 uint16_t flags; 502 uint16_t idx; 503 VRingUsedElem ring[]; 504 } VRingUsed; 505 506 typedef struct VRingMemoryRegionCaches { 507 struct rcu_head rcu; 508 MemoryRegionCache desc; 509 MemoryRegionCache avail; 510 MemoryRegionCache used; 511 } VRingMemoryRegionCaches; 512 513 typedef struct VRing 514 { 515 unsigned int num; 516 unsigned int num_default; 517 unsigned int align; 518 hwaddr desc; 519 hwaddr avail; 520 hwaddr used; 521 VRingMemoryRegionCaches *caches; 522 } VRing; 523 524 typedef struct VRingPackedDescEvent { 525 uint16_t off_wrap; 526 uint16_t flags; 527 } VRingPackedDescEvent ; 528 529 struct VirtQueue 530 { 531 VRing vring; 532 VirtQueueElement *used_elems; 533 534 /* Next head to pop */ 535 uint16_t last_avail_idx; 536 bool last_avail_wrap_counter; 537 538 /* Last avail_idx read from VQ. */ 539 uint16_t shadow_avail_idx; 540 bool shadow_avail_wrap_counter; 541 542 uint16_t used_idx; 543 bool used_wrap_counter; 544 545 /* Last used index value we have signalled on */ 546 uint16_t signalled_used; 547 548 /* Last used index value we have signalled on */ 549 bool signalled_used_valid; 550 551 /* Notification enabled? */ 552 bool notification; 553 554 uint16_t queue_index; 555 556 unsigned int inuse; 557 558 uint16_t vector; 559 VirtIOHandleOutput handle_output; 560 VirtIODevice *vdev; 561 EventNotifier guest_notifier; 562 EventNotifier host_notifier; 563 bool host_notifier_enabled; 564 QLIST_ENTRY(VirtQueue) node; 565 }; 566 567 const char *virtio_device_names[] = { 568 [VIRTIO_ID_NET] = "virtio-net", 569 [VIRTIO_ID_BLOCK] = "virtio-blk", 570 [VIRTIO_ID_CONSOLE] = "virtio-serial", 571 [VIRTIO_ID_RNG] = "virtio-rng", 572 [VIRTIO_ID_BALLOON] = "virtio-balloon", 573 [VIRTIO_ID_IOMEM] = "virtio-iomem", 574 [VIRTIO_ID_RPMSG] = "virtio-rpmsg", 575 [VIRTIO_ID_SCSI] = "virtio-scsi", 576 [VIRTIO_ID_9P] = "virtio-9p", 577 [VIRTIO_ID_MAC80211_WLAN] = "virtio-mac-wlan", 578 [VIRTIO_ID_RPROC_SERIAL] = "virtio-rproc-serial", 579 [VIRTIO_ID_CAIF] = "virtio-caif", 580 [VIRTIO_ID_MEMORY_BALLOON] = "virtio-mem-balloon", 581 [VIRTIO_ID_GPU] = "virtio-gpu", 582 [VIRTIO_ID_CLOCK] = "virtio-clk", 583 [VIRTIO_ID_INPUT] = "virtio-input", 584 [VIRTIO_ID_VSOCK] = "vhost-vsock", 585 [VIRTIO_ID_CRYPTO] = "virtio-crypto", 586 [VIRTIO_ID_SIGNAL_DIST] = "virtio-signal", 587 [VIRTIO_ID_PSTORE] = "virtio-pstore", 588 [VIRTIO_ID_IOMMU] = "virtio-iommu", 589 [VIRTIO_ID_MEM] = "virtio-mem", 590 [VIRTIO_ID_SOUND] = "virtio-sound", 591 [VIRTIO_ID_FS] = "virtio-user-fs", 592 [VIRTIO_ID_PMEM] = "virtio-pmem", 593 [VIRTIO_ID_RPMB] = "virtio-rpmb", 594 [VIRTIO_ID_MAC80211_HWSIM] = "virtio-mac-hwsim", 595 [VIRTIO_ID_VIDEO_ENCODER] = "virtio-vid-encoder", 596 [VIRTIO_ID_VIDEO_DECODER] = "virtio-vid-decoder", 597 [VIRTIO_ID_SCMI] = "virtio-scmi", 598 [VIRTIO_ID_NITRO_SEC_MOD] = "virtio-nitro-sec-mod", 599 [VIRTIO_ID_I2C_ADAPTER] = "vhost-user-i2c", 600 [VIRTIO_ID_WATCHDOG] = "virtio-watchdog", 601 [VIRTIO_ID_CAN] = "virtio-can", 602 [VIRTIO_ID_DMABUF] = "virtio-dmabuf", 603 [VIRTIO_ID_PARAM_SERV] = "virtio-param-serv", 604 [VIRTIO_ID_AUDIO_POLICY] = "virtio-audio-pol", 605 [VIRTIO_ID_BT] = "virtio-bluetooth", 606 [VIRTIO_ID_GPIO] = "virtio-gpio" 607 }; 608 609 static const char *virtio_id_to_name(uint16_t device_id) 610 { 611 assert(device_id < G_N_ELEMENTS(virtio_device_names)); 612 const char *name = virtio_device_names[device_id]; 613 assert(name != NULL); 614 return name; 615 } 616 617 /* Called within call_rcu(). */ 618 static void virtio_free_region_cache(VRingMemoryRegionCaches *caches) 619 { 620 assert(caches != NULL); 621 address_space_cache_destroy(&caches->desc); 622 address_space_cache_destroy(&caches->avail); 623 address_space_cache_destroy(&caches->used); 624 g_free(caches); 625 } 626 627 static void virtio_virtqueue_reset_region_cache(struct VirtQueue *vq) 628 { 629 VRingMemoryRegionCaches *caches; 630 631 caches = qatomic_read(&vq->vring.caches); 632 qatomic_rcu_set(&vq->vring.caches, NULL); 633 if (caches) { 634 call_rcu(caches, virtio_free_region_cache, rcu); 635 } 636 } 637 638 static void virtio_init_region_cache(VirtIODevice *vdev, int n) 639 { 640 VirtQueue *vq = &vdev->vq[n]; 641 VRingMemoryRegionCaches *old = vq->vring.caches; 642 VRingMemoryRegionCaches *new = NULL; 643 hwaddr addr, size; 644 int64_t len; 645 bool packed; 646 647 648 addr = vq->vring.desc; 649 if (!addr) { 650 goto out_no_cache; 651 } 652 new = g_new0(VRingMemoryRegionCaches, 1); 653 size = virtio_queue_get_desc_size(vdev, n); 654 packed = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ? 655 true : false; 656 len = address_space_cache_init(&new->desc, vdev->dma_as, 657 addr, size, packed); 658 if (len < size) { 659 virtio_error(vdev, "Cannot map desc"); 660 goto err_desc; 661 } 662 663 size = virtio_queue_get_used_size(vdev, n); 664 len = address_space_cache_init(&new->used, vdev->dma_as, 665 vq->vring.used, size, true); 666 if (len < size) { 667 virtio_error(vdev, "Cannot map used"); 668 goto err_used; 669 } 670 671 size = virtio_queue_get_avail_size(vdev, n); 672 len = address_space_cache_init(&new->avail, vdev->dma_as, 673 vq->vring.avail, size, false); 674 if (len < size) { 675 virtio_error(vdev, "Cannot map avail"); 676 goto err_avail; 677 } 678 679 qatomic_rcu_set(&vq->vring.caches, new); 680 if (old) { 681 call_rcu(old, virtio_free_region_cache, rcu); 682 } 683 return; 684 685 err_avail: 686 address_space_cache_destroy(&new->avail); 687 err_used: 688 address_space_cache_destroy(&new->used); 689 err_desc: 690 address_space_cache_destroy(&new->desc); 691 out_no_cache: 692 g_free(new); 693 virtio_virtqueue_reset_region_cache(vq); 694 } 695 696 /* virt queue functions */ 697 void virtio_queue_update_rings(VirtIODevice *vdev, int n) 698 { 699 VRing *vring = &vdev->vq[n].vring; 700 701 if (!vring->num || !vring->desc || !vring->align) { 702 /* not yet setup -> nothing to do */ 703 return; 704 } 705 vring->avail = vring->desc + vring->num * sizeof(VRingDesc); 706 vring->used = vring_align(vring->avail + 707 offsetof(VRingAvail, ring[vring->num]), 708 vring->align); 709 virtio_init_region_cache(vdev, n); 710 } 711 712 /* Called within rcu_read_lock(). */ 713 static void vring_split_desc_read(VirtIODevice *vdev, VRingDesc *desc, 714 MemoryRegionCache *cache, int i) 715 { 716 address_space_read_cached(cache, i * sizeof(VRingDesc), 717 desc, sizeof(VRingDesc)); 718 virtio_tswap64s(vdev, &desc->addr); 719 virtio_tswap32s(vdev, &desc->len); 720 virtio_tswap16s(vdev, &desc->flags); 721 virtio_tswap16s(vdev, &desc->next); 722 } 723 724 static void vring_packed_event_read(VirtIODevice *vdev, 725 MemoryRegionCache *cache, 726 VRingPackedDescEvent *e) 727 { 728 hwaddr off_off = offsetof(VRingPackedDescEvent, off_wrap); 729 hwaddr off_flags = offsetof(VRingPackedDescEvent, flags); 730 731 e->flags = virtio_lduw_phys_cached(vdev, cache, off_flags); 732 /* Make sure flags is seen before off_wrap */ 733 smp_rmb(); 734 e->off_wrap = virtio_lduw_phys_cached(vdev, cache, off_off); 735 virtio_tswap16s(vdev, &e->flags); 736 } 737 738 static void vring_packed_off_wrap_write(VirtIODevice *vdev, 739 MemoryRegionCache *cache, 740 uint16_t off_wrap) 741 { 742 hwaddr off = offsetof(VRingPackedDescEvent, off_wrap); 743 744 virtio_stw_phys_cached(vdev, cache, off, off_wrap); 745 address_space_cache_invalidate(cache, off, sizeof(off_wrap)); 746 } 747 748 static void vring_packed_flags_write(VirtIODevice *vdev, 749 MemoryRegionCache *cache, uint16_t flags) 750 { 751 hwaddr off = offsetof(VRingPackedDescEvent, flags); 752 753 virtio_stw_phys_cached(vdev, cache, off, flags); 754 address_space_cache_invalidate(cache, off, sizeof(flags)); 755 } 756 757 /* Called within rcu_read_lock(). */ 758 static VRingMemoryRegionCaches *vring_get_region_caches(struct VirtQueue *vq) 759 { 760 return qatomic_rcu_read(&vq->vring.caches); 761 } 762 763 /* Called within rcu_read_lock(). */ 764 static inline uint16_t vring_avail_flags(VirtQueue *vq) 765 { 766 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 767 hwaddr pa = offsetof(VRingAvail, flags); 768 769 if (!caches) { 770 return 0; 771 } 772 773 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); 774 } 775 776 /* Called within rcu_read_lock(). */ 777 static inline uint16_t vring_avail_idx(VirtQueue *vq) 778 { 779 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 780 hwaddr pa = offsetof(VRingAvail, idx); 781 782 if (!caches) { 783 return 0; 784 } 785 786 vq->shadow_avail_idx = virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); 787 return vq->shadow_avail_idx; 788 } 789 790 /* Called within rcu_read_lock(). */ 791 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i) 792 { 793 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 794 hwaddr pa = offsetof(VRingAvail, ring[i]); 795 796 if (!caches) { 797 return 0; 798 } 799 800 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); 801 } 802 803 /* Called within rcu_read_lock(). */ 804 static inline uint16_t vring_get_used_event(VirtQueue *vq) 805 { 806 return vring_avail_ring(vq, vq->vring.num); 807 } 808 809 /* Called within rcu_read_lock(). */ 810 static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem, 811 int i) 812 { 813 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 814 hwaddr pa = offsetof(VRingUsed, ring[i]); 815 816 if (!caches) { 817 return; 818 } 819 820 virtio_tswap32s(vq->vdev, &uelem->id); 821 virtio_tswap32s(vq->vdev, &uelem->len); 822 address_space_write_cached(&caches->used, pa, uelem, sizeof(VRingUsedElem)); 823 address_space_cache_invalidate(&caches->used, pa, sizeof(VRingUsedElem)); 824 } 825 826 /* Called within rcu_read_lock(). */ 827 static inline uint16_t vring_used_flags(VirtQueue *vq) 828 { 829 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 830 hwaddr pa = offsetof(VRingUsed, flags); 831 832 if (!caches) { 833 return 0; 834 } 835 836 return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); 837 } 838 839 /* Called within rcu_read_lock(). */ 840 static uint16_t vring_used_idx(VirtQueue *vq) 841 { 842 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 843 hwaddr pa = offsetof(VRingUsed, idx); 844 845 if (!caches) { 846 return 0; 847 } 848 849 return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); 850 } 851 852 /* Called within rcu_read_lock(). */ 853 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val) 854 { 855 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 856 hwaddr pa = offsetof(VRingUsed, idx); 857 858 if (caches) { 859 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val); 860 address_space_cache_invalidate(&caches->used, pa, sizeof(val)); 861 } 862 863 vq->used_idx = val; 864 } 865 866 /* Called within rcu_read_lock(). */ 867 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask) 868 { 869 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 870 VirtIODevice *vdev = vq->vdev; 871 hwaddr pa = offsetof(VRingUsed, flags); 872 uint16_t flags; 873 874 if (!caches) { 875 return; 876 } 877 878 flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); 879 virtio_stw_phys_cached(vdev, &caches->used, pa, flags | mask); 880 address_space_cache_invalidate(&caches->used, pa, sizeof(flags)); 881 } 882 883 /* Called within rcu_read_lock(). */ 884 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask) 885 { 886 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 887 VirtIODevice *vdev = vq->vdev; 888 hwaddr pa = offsetof(VRingUsed, flags); 889 uint16_t flags; 890 891 if (!caches) { 892 return; 893 } 894 895 flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); 896 virtio_stw_phys_cached(vdev, &caches->used, pa, flags & ~mask); 897 address_space_cache_invalidate(&caches->used, pa, sizeof(flags)); 898 } 899 900 /* Called within rcu_read_lock(). */ 901 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val) 902 { 903 VRingMemoryRegionCaches *caches; 904 hwaddr pa; 905 if (!vq->notification) { 906 return; 907 } 908 909 caches = vring_get_region_caches(vq); 910 if (!caches) { 911 return; 912 } 913 914 pa = offsetof(VRingUsed, ring[vq->vring.num]); 915 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val); 916 address_space_cache_invalidate(&caches->used, pa, sizeof(val)); 917 } 918 919 static void virtio_queue_split_set_notification(VirtQueue *vq, int enable) 920 { 921 RCU_READ_LOCK_GUARD(); 922 923 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { 924 vring_set_avail_event(vq, vring_avail_idx(vq)); 925 } else if (enable) { 926 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY); 927 } else { 928 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY); 929 } 930 if (enable) { 931 /* Expose avail event/used flags before caller checks the avail idx. */ 932 smp_mb(); 933 } 934 } 935 936 static void virtio_queue_packed_set_notification(VirtQueue *vq, int enable) 937 { 938 uint16_t off_wrap; 939 VRingPackedDescEvent e; 940 VRingMemoryRegionCaches *caches; 941 942 RCU_READ_LOCK_GUARD(); 943 caches = vring_get_region_caches(vq); 944 if (!caches) { 945 return; 946 } 947 948 vring_packed_event_read(vq->vdev, &caches->used, &e); 949 950 if (!enable) { 951 e.flags = VRING_PACKED_EVENT_FLAG_DISABLE; 952 } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { 953 off_wrap = vq->shadow_avail_idx | vq->shadow_avail_wrap_counter << 15; 954 vring_packed_off_wrap_write(vq->vdev, &caches->used, off_wrap); 955 /* Make sure off_wrap is wrote before flags */ 956 smp_wmb(); 957 e.flags = VRING_PACKED_EVENT_FLAG_DESC; 958 } else { 959 e.flags = VRING_PACKED_EVENT_FLAG_ENABLE; 960 } 961 962 vring_packed_flags_write(vq->vdev, &caches->used, e.flags); 963 if (enable) { 964 /* Expose avail event/used flags before caller checks the avail idx. */ 965 smp_mb(); 966 } 967 } 968 969 bool virtio_queue_get_notification(VirtQueue *vq) 970 { 971 return vq->notification; 972 } 973 974 void virtio_queue_set_notification(VirtQueue *vq, int enable) 975 { 976 vq->notification = enable; 977 978 if (!vq->vring.desc) { 979 return; 980 } 981 982 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 983 virtio_queue_packed_set_notification(vq, enable); 984 } else { 985 virtio_queue_split_set_notification(vq, enable); 986 } 987 } 988 989 int virtio_queue_ready(VirtQueue *vq) 990 { 991 return vq->vring.avail != 0; 992 } 993 994 static void vring_packed_desc_read_flags(VirtIODevice *vdev, 995 uint16_t *flags, 996 MemoryRegionCache *cache, 997 int i) 998 { 999 hwaddr off = i * sizeof(VRingPackedDesc) + offsetof(VRingPackedDesc, flags); 1000 1001 *flags = virtio_lduw_phys_cached(vdev, cache, off); 1002 } 1003 1004 static void vring_packed_desc_read(VirtIODevice *vdev, 1005 VRingPackedDesc *desc, 1006 MemoryRegionCache *cache, 1007 int i, bool strict_order) 1008 { 1009 hwaddr off = i * sizeof(VRingPackedDesc); 1010 1011 vring_packed_desc_read_flags(vdev, &desc->flags, cache, i); 1012 1013 if (strict_order) { 1014 /* Make sure flags is read before the rest fields. */ 1015 smp_rmb(); 1016 } 1017 1018 address_space_read_cached(cache, off + offsetof(VRingPackedDesc, addr), 1019 &desc->addr, sizeof(desc->addr)); 1020 address_space_read_cached(cache, off + offsetof(VRingPackedDesc, id), 1021 &desc->id, sizeof(desc->id)); 1022 address_space_read_cached(cache, off + offsetof(VRingPackedDesc, len), 1023 &desc->len, sizeof(desc->len)); 1024 virtio_tswap64s(vdev, &desc->addr); 1025 virtio_tswap16s(vdev, &desc->id); 1026 virtio_tswap32s(vdev, &desc->len); 1027 } 1028 1029 static void vring_packed_desc_write_data(VirtIODevice *vdev, 1030 VRingPackedDesc *desc, 1031 MemoryRegionCache *cache, 1032 int i) 1033 { 1034 hwaddr off_id = i * sizeof(VRingPackedDesc) + 1035 offsetof(VRingPackedDesc, id); 1036 hwaddr off_len = i * sizeof(VRingPackedDesc) + 1037 offsetof(VRingPackedDesc, len); 1038 1039 virtio_tswap32s(vdev, &desc->len); 1040 virtio_tswap16s(vdev, &desc->id); 1041 address_space_write_cached(cache, off_id, &desc->id, sizeof(desc->id)); 1042 address_space_cache_invalidate(cache, off_id, sizeof(desc->id)); 1043 address_space_write_cached(cache, off_len, &desc->len, sizeof(desc->len)); 1044 address_space_cache_invalidate(cache, off_len, sizeof(desc->len)); 1045 } 1046 1047 static void vring_packed_desc_write_flags(VirtIODevice *vdev, 1048 VRingPackedDesc *desc, 1049 MemoryRegionCache *cache, 1050 int i) 1051 { 1052 hwaddr off = i * sizeof(VRingPackedDesc) + offsetof(VRingPackedDesc, flags); 1053 1054 virtio_stw_phys_cached(vdev, cache, off, desc->flags); 1055 address_space_cache_invalidate(cache, off, sizeof(desc->flags)); 1056 } 1057 1058 static void vring_packed_desc_write(VirtIODevice *vdev, 1059 VRingPackedDesc *desc, 1060 MemoryRegionCache *cache, 1061 int i, bool strict_order) 1062 { 1063 vring_packed_desc_write_data(vdev, desc, cache, i); 1064 if (strict_order) { 1065 /* Make sure data is wrote before flags. */ 1066 smp_wmb(); 1067 } 1068 vring_packed_desc_write_flags(vdev, desc, cache, i); 1069 } 1070 1071 static inline bool is_desc_avail(uint16_t flags, bool wrap_counter) 1072 { 1073 bool avail, used; 1074 1075 avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL)); 1076 used = !!(flags & (1 << VRING_PACKED_DESC_F_USED)); 1077 return (avail != used) && (avail == wrap_counter); 1078 } 1079 1080 /* Fetch avail_idx from VQ memory only when we really need to know if 1081 * guest has added some buffers. 1082 * Called within rcu_read_lock(). */ 1083 static int virtio_queue_empty_rcu(VirtQueue *vq) 1084 { 1085 if (virtio_device_disabled(vq->vdev)) { 1086 return 1; 1087 } 1088 1089 if (unlikely(!vq->vring.avail)) { 1090 return 1; 1091 } 1092 1093 if (vq->shadow_avail_idx != vq->last_avail_idx) { 1094 return 0; 1095 } 1096 1097 return vring_avail_idx(vq) == vq->last_avail_idx; 1098 } 1099 1100 static int virtio_queue_split_empty(VirtQueue *vq) 1101 { 1102 bool empty; 1103 1104 if (virtio_device_disabled(vq->vdev)) { 1105 return 1; 1106 } 1107 1108 if (unlikely(!vq->vring.avail)) { 1109 return 1; 1110 } 1111 1112 if (vq->shadow_avail_idx != vq->last_avail_idx) { 1113 return 0; 1114 } 1115 1116 RCU_READ_LOCK_GUARD(); 1117 empty = vring_avail_idx(vq) == vq->last_avail_idx; 1118 return empty; 1119 } 1120 1121 /* Called within rcu_read_lock(). */ 1122 static int virtio_queue_packed_empty_rcu(VirtQueue *vq) 1123 { 1124 struct VRingPackedDesc desc; 1125 VRingMemoryRegionCaches *cache; 1126 1127 if (unlikely(!vq->vring.desc)) { 1128 return 1; 1129 } 1130 1131 cache = vring_get_region_caches(vq); 1132 if (!cache) { 1133 return 1; 1134 } 1135 1136 vring_packed_desc_read_flags(vq->vdev, &desc.flags, &cache->desc, 1137 vq->last_avail_idx); 1138 1139 return !is_desc_avail(desc.flags, vq->last_avail_wrap_counter); 1140 } 1141 1142 static int virtio_queue_packed_empty(VirtQueue *vq) 1143 { 1144 RCU_READ_LOCK_GUARD(); 1145 return virtio_queue_packed_empty_rcu(vq); 1146 } 1147 1148 int virtio_queue_empty(VirtQueue *vq) 1149 { 1150 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1151 return virtio_queue_packed_empty(vq); 1152 } else { 1153 return virtio_queue_split_empty(vq); 1154 } 1155 } 1156 1157 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem, 1158 unsigned int len) 1159 { 1160 AddressSpace *dma_as = vq->vdev->dma_as; 1161 unsigned int offset; 1162 int i; 1163 1164 offset = 0; 1165 for (i = 0; i < elem->in_num; i++) { 1166 size_t size = MIN(len - offset, elem->in_sg[i].iov_len); 1167 1168 dma_memory_unmap(dma_as, elem->in_sg[i].iov_base, 1169 elem->in_sg[i].iov_len, 1170 DMA_DIRECTION_FROM_DEVICE, size); 1171 1172 offset += size; 1173 } 1174 1175 for (i = 0; i < elem->out_num; i++) 1176 dma_memory_unmap(dma_as, elem->out_sg[i].iov_base, 1177 elem->out_sg[i].iov_len, 1178 DMA_DIRECTION_TO_DEVICE, 1179 elem->out_sg[i].iov_len); 1180 } 1181 1182 /* virtqueue_detach_element: 1183 * @vq: The #VirtQueue 1184 * @elem: The #VirtQueueElement 1185 * @len: number of bytes written 1186 * 1187 * Detach the element from the virtqueue. This function is suitable for device 1188 * reset or other situations where a #VirtQueueElement is simply freed and will 1189 * not be pushed or discarded. 1190 */ 1191 void virtqueue_detach_element(VirtQueue *vq, const VirtQueueElement *elem, 1192 unsigned int len) 1193 { 1194 vq->inuse -= elem->ndescs; 1195 virtqueue_unmap_sg(vq, elem, len); 1196 } 1197 1198 static void virtqueue_split_rewind(VirtQueue *vq, unsigned int num) 1199 { 1200 vq->last_avail_idx -= num; 1201 } 1202 1203 static void virtqueue_packed_rewind(VirtQueue *vq, unsigned int num) 1204 { 1205 if (vq->last_avail_idx < num) { 1206 vq->last_avail_idx = vq->vring.num + vq->last_avail_idx - num; 1207 vq->last_avail_wrap_counter ^= 1; 1208 } else { 1209 vq->last_avail_idx -= num; 1210 } 1211 } 1212 1213 /* virtqueue_unpop: 1214 * @vq: The #VirtQueue 1215 * @elem: The #VirtQueueElement 1216 * @len: number of bytes written 1217 * 1218 * Pretend the most recent element wasn't popped from the virtqueue. The next 1219 * call to virtqueue_pop() will refetch the element. 1220 */ 1221 void virtqueue_unpop(VirtQueue *vq, const VirtQueueElement *elem, 1222 unsigned int len) 1223 { 1224 1225 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1226 virtqueue_packed_rewind(vq, 1); 1227 } else { 1228 virtqueue_split_rewind(vq, 1); 1229 } 1230 1231 virtqueue_detach_element(vq, elem, len); 1232 } 1233 1234 /* virtqueue_rewind: 1235 * @vq: The #VirtQueue 1236 * @num: Number of elements to push back 1237 * 1238 * Pretend that elements weren't popped from the virtqueue. The next 1239 * virtqueue_pop() will refetch the oldest element. 1240 * 1241 * Use virtqueue_unpop() instead if you have a VirtQueueElement. 1242 * 1243 * Returns: true on success, false if @num is greater than the number of in use 1244 * elements. 1245 */ 1246 bool virtqueue_rewind(VirtQueue *vq, unsigned int num) 1247 { 1248 if (num > vq->inuse) { 1249 return false; 1250 } 1251 1252 vq->inuse -= num; 1253 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1254 virtqueue_packed_rewind(vq, num); 1255 } else { 1256 virtqueue_split_rewind(vq, num); 1257 } 1258 return true; 1259 } 1260 1261 static void virtqueue_split_fill(VirtQueue *vq, const VirtQueueElement *elem, 1262 unsigned int len, unsigned int idx) 1263 { 1264 VRingUsedElem uelem; 1265 1266 if (unlikely(!vq->vring.used)) { 1267 return; 1268 } 1269 1270 idx = (idx + vq->used_idx) % vq->vring.num; 1271 1272 uelem.id = elem->index; 1273 uelem.len = len; 1274 vring_used_write(vq, &uelem, idx); 1275 } 1276 1277 static void virtqueue_packed_fill(VirtQueue *vq, const VirtQueueElement *elem, 1278 unsigned int len, unsigned int idx) 1279 { 1280 vq->used_elems[idx].index = elem->index; 1281 vq->used_elems[idx].len = len; 1282 vq->used_elems[idx].ndescs = elem->ndescs; 1283 } 1284 1285 static void virtqueue_packed_fill_desc(VirtQueue *vq, 1286 const VirtQueueElement *elem, 1287 unsigned int idx, 1288 bool strict_order) 1289 { 1290 uint16_t head; 1291 VRingMemoryRegionCaches *caches; 1292 VRingPackedDesc desc = { 1293 .id = elem->index, 1294 .len = elem->len, 1295 }; 1296 bool wrap_counter = vq->used_wrap_counter; 1297 1298 if (unlikely(!vq->vring.desc)) { 1299 return; 1300 } 1301 1302 head = vq->used_idx + idx; 1303 if (head >= vq->vring.num) { 1304 head -= vq->vring.num; 1305 wrap_counter ^= 1; 1306 } 1307 if (wrap_counter) { 1308 desc.flags |= (1 << VRING_PACKED_DESC_F_AVAIL); 1309 desc.flags |= (1 << VRING_PACKED_DESC_F_USED); 1310 } else { 1311 desc.flags &= ~(1 << VRING_PACKED_DESC_F_AVAIL); 1312 desc.flags &= ~(1 << VRING_PACKED_DESC_F_USED); 1313 } 1314 1315 caches = vring_get_region_caches(vq); 1316 if (!caches) { 1317 return; 1318 } 1319 1320 vring_packed_desc_write(vq->vdev, &desc, &caches->desc, head, strict_order); 1321 } 1322 1323 /* Called within rcu_read_lock(). */ 1324 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem, 1325 unsigned int len, unsigned int idx) 1326 { 1327 trace_virtqueue_fill(vq, elem, len, idx); 1328 1329 virtqueue_unmap_sg(vq, elem, len); 1330 1331 if (virtio_device_disabled(vq->vdev)) { 1332 return; 1333 } 1334 1335 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1336 virtqueue_packed_fill(vq, elem, len, idx); 1337 } else { 1338 virtqueue_split_fill(vq, elem, len, idx); 1339 } 1340 } 1341 1342 /* Called within rcu_read_lock(). */ 1343 static void virtqueue_split_flush(VirtQueue *vq, unsigned int count) 1344 { 1345 uint16_t old, new; 1346 1347 if (unlikely(!vq->vring.used)) { 1348 return; 1349 } 1350 1351 /* Make sure buffer is written before we update index. */ 1352 smp_wmb(); 1353 trace_virtqueue_flush(vq, count); 1354 old = vq->used_idx; 1355 new = old + count; 1356 vring_used_idx_set(vq, new); 1357 vq->inuse -= count; 1358 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old))) 1359 vq->signalled_used_valid = false; 1360 } 1361 1362 static void virtqueue_packed_flush(VirtQueue *vq, unsigned int count) 1363 { 1364 unsigned int i, ndescs = 0; 1365 1366 if (unlikely(!vq->vring.desc)) { 1367 return; 1368 } 1369 1370 for (i = 1; i < count; i++) { 1371 virtqueue_packed_fill_desc(vq, &vq->used_elems[i], i, false); 1372 ndescs += vq->used_elems[i].ndescs; 1373 } 1374 virtqueue_packed_fill_desc(vq, &vq->used_elems[0], 0, true); 1375 ndescs += vq->used_elems[0].ndescs; 1376 1377 vq->inuse -= ndescs; 1378 vq->used_idx += ndescs; 1379 if (vq->used_idx >= vq->vring.num) { 1380 vq->used_idx -= vq->vring.num; 1381 vq->used_wrap_counter ^= 1; 1382 vq->signalled_used_valid = false; 1383 } 1384 } 1385 1386 void virtqueue_flush(VirtQueue *vq, unsigned int count) 1387 { 1388 if (virtio_device_disabled(vq->vdev)) { 1389 vq->inuse -= count; 1390 return; 1391 } 1392 1393 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1394 virtqueue_packed_flush(vq, count); 1395 } else { 1396 virtqueue_split_flush(vq, count); 1397 } 1398 } 1399 1400 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem, 1401 unsigned int len) 1402 { 1403 RCU_READ_LOCK_GUARD(); 1404 virtqueue_fill(vq, elem, len, 0); 1405 virtqueue_flush(vq, 1); 1406 } 1407 1408 /* Called within rcu_read_lock(). */ 1409 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx) 1410 { 1411 uint16_t num_heads = vring_avail_idx(vq) - idx; 1412 1413 /* Check it isn't doing very strange things with descriptor numbers. */ 1414 if (num_heads > vq->vring.num) { 1415 virtio_error(vq->vdev, "Guest moved used index from %u to %u", 1416 idx, vq->shadow_avail_idx); 1417 return -EINVAL; 1418 } 1419 /* On success, callers read a descriptor at vq->last_avail_idx. 1420 * Make sure descriptor read does not bypass avail index read. */ 1421 if (num_heads) { 1422 smp_rmb(); 1423 } 1424 1425 return num_heads; 1426 } 1427 1428 /* Called within rcu_read_lock(). */ 1429 static bool virtqueue_get_head(VirtQueue *vq, unsigned int idx, 1430 unsigned int *head) 1431 { 1432 /* Grab the next descriptor number they're advertising, and increment 1433 * the index we've seen. */ 1434 *head = vring_avail_ring(vq, idx % vq->vring.num); 1435 1436 /* If their number is silly, that's a fatal mistake. */ 1437 if (*head >= vq->vring.num) { 1438 virtio_error(vq->vdev, "Guest says index %u is available", *head); 1439 return false; 1440 } 1441 1442 return true; 1443 } 1444 1445 enum { 1446 VIRTQUEUE_READ_DESC_ERROR = -1, 1447 VIRTQUEUE_READ_DESC_DONE = 0, /* end of chain */ 1448 VIRTQUEUE_READ_DESC_MORE = 1, /* more buffers in chain */ 1449 }; 1450 1451 static int virtqueue_split_read_next_desc(VirtIODevice *vdev, VRingDesc *desc, 1452 MemoryRegionCache *desc_cache, 1453 unsigned int max, unsigned int *next) 1454 { 1455 /* If this descriptor says it doesn't chain, we're done. */ 1456 if (!(desc->flags & VRING_DESC_F_NEXT)) { 1457 return VIRTQUEUE_READ_DESC_DONE; 1458 } 1459 1460 /* Check they're not leading us off end of descriptors. */ 1461 *next = desc->next; 1462 /* Make sure compiler knows to grab that: we don't want it changing! */ 1463 smp_wmb(); 1464 1465 if (*next >= max) { 1466 virtio_error(vdev, "Desc next is %u", *next); 1467 return VIRTQUEUE_READ_DESC_ERROR; 1468 } 1469 1470 vring_split_desc_read(vdev, desc, desc_cache, *next); 1471 return VIRTQUEUE_READ_DESC_MORE; 1472 } 1473 1474 /* Called within rcu_read_lock(). */ 1475 static void virtqueue_split_get_avail_bytes(VirtQueue *vq, 1476 unsigned int *in_bytes, unsigned int *out_bytes, 1477 unsigned max_in_bytes, unsigned max_out_bytes, 1478 VRingMemoryRegionCaches *caches) 1479 { 1480 VirtIODevice *vdev = vq->vdev; 1481 unsigned int max, idx; 1482 unsigned int total_bufs, in_total, out_total; 1483 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 1484 int64_t len = 0; 1485 int rc; 1486 1487 idx = vq->last_avail_idx; 1488 total_bufs = in_total = out_total = 0; 1489 1490 max = vq->vring.num; 1491 1492 while ((rc = virtqueue_num_heads(vq, idx)) > 0) { 1493 MemoryRegionCache *desc_cache = &caches->desc; 1494 unsigned int num_bufs; 1495 VRingDesc desc; 1496 unsigned int i; 1497 1498 num_bufs = total_bufs; 1499 1500 if (!virtqueue_get_head(vq, idx++, &i)) { 1501 goto err; 1502 } 1503 1504 vring_split_desc_read(vdev, &desc, desc_cache, i); 1505 1506 if (desc.flags & VRING_DESC_F_INDIRECT) { 1507 if (!desc.len || (desc.len % sizeof(VRingDesc))) { 1508 virtio_error(vdev, "Invalid size for indirect buffer table"); 1509 goto err; 1510 } 1511 1512 /* If we've got too many, that implies a descriptor loop. */ 1513 if (num_bufs >= max) { 1514 virtio_error(vdev, "Looped descriptor"); 1515 goto err; 1516 } 1517 1518 /* loop over the indirect descriptor table */ 1519 len = address_space_cache_init(&indirect_desc_cache, 1520 vdev->dma_as, 1521 desc.addr, desc.len, false); 1522 desc_cache = &indirect_desc_cache; 1523 if (len < desc.len) { 1524 virtio_error(vdev, "Cannot map indirect buffer"); 1525 goto err; 1526 } 1527 1528 max = desc.len / sizeof(VRingDesc); 1529 num_bufs = i = 0; 1530 vring_split_desc_read(vdev, &desc, desc_cache, i); 1531 } 1532 1533 do { 1534 /* If we've got too many, that implies a descriptor loop. */ 1535 if (++num_bufs > max) { 1536 virtio_error(vdev, "Looped descriptor"); 1537 goto err; 1538 } 1539 1540 if (desc.flags & VRING_DESC_F_WRITE) { 1541 in_total += desc.len; 1542 } else { 1543 out_total += desc.len; 1544 } 1545 if (in_total >= max_in_bytes && out_total >= max_out_bytes) { 1546 goto done; 1547 } 1548 1549 rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max, &i); 1550 } while (rc == VIRTQUEUE_READ_DESC_MORE); 1551 1552 if (rc == VIRTQUEUE_READ_DESC_ERROR) { 1553 goto err; 1554 } 1555 1556 if (desc_cache == &indirect_desc_cache) { 1557 address_space_cache_destroy(&indirect_desc_cache); 1558 total_bufs++; 1559 } else { 1560 total_bufs = num_bufs; 1561 } 1562 } 1563 1564 if (rc < 0) { 1565 goto err; 1566 } 1567 1568 done: 1569 address_space_cache_destroy(&indirect_desc_cache); 1570 if (in_bytes) { 1571 *in_bytes = in_total; 1572 } 1573 if (out_bytes) { 1574 *out_bytes = out_total; 1575 } 1576 return; 1577 1578 err: 1579 in_total = out_total = 0; 1580 goto done; 1581 } 1582 1583 static int virtqueue_packed_read_next_desc(VirtQueue *vq, 1584 VRingPackedDesc *desc, 1585 MemoryRegionCache 1586 *desc_cache, 1587 unsigned int max, 1588 unsigned int *next, 1589 bool indirect) 1590 { 1591 /* If this descriptor says it doesn't chain, we're done. */ 1592 if (!indirect && !(desc->flags & VRING_DESC_F_NEXT)) { 1593 return VIRTQUEUE_READ_DESC_DONE; 1594 } 1595 1596 ++*next; 1597 if (*next == max) { 1598 if (indirect) { 1599 return VIRTQUEUE_READ_DESC_DONE; 1600 } else { 1601 (*next) -= vq->vring.num; 1602 } 1603 } 1604 1605 vring_packed_desc_read(vq->vdev, desc, desc_cache, *next, false); 1606 return VIRTQUEUE_READ_DESC_MORE; 1607 } 1608 1609 /* Called within rcu_read_lock(). */ 1610 static void virtqueue_packed_get_avail_bytes(VirtQueue *vq, 1611 unsigned int *in_bytes, 1612 unsigned int *out_bytes, 1613 unsigned max_in_bytes, 1614 unsigned max_out_bytes, 1615 VRingMemoryRegionCaches *caches) 1616 { 1617 VirtIODevice *vdev = vq->vdev; 1618 unsigned int max, idx; 1619 unsigned int total_bufs, in_total, out_total; 1620 MemoryRegionCache *desc_cache; 1621 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 1622 int64_t len = 0; 1623 VRingPackedDesc desc; 1624 bool wrap_counter; 1625 1626 idx = vq->last_avail_idx; 1627 wrap_counter = vq->last_avail_wrap_counter; 1628 total_bufs = in_total = out_total = 0; 1629 1630 max = vq->vring.num; 1631 1632 for (;;) { 1633 unsigned int num_bufs = total_bufs; 1634 unsigned int i = idx; 1635 int rc; 1636 1637 desc_cache = &caches->desc; 1638 vring_packed_desc_read(vdev, &desc, desc_cache, idx, true); 1639 if (!is_desc_avail(desc.flags, wrap_counter)) { 1640 break; 1641 } 1642 1643 if (desc.flags & VRING_DESC_F_INDIRECT) { 1644 if (desc.len % sizeof(VRingPackedDesc)) { 1645 virtio_error(vdev, "Invalid size for indirect buffer table"); 1646 goto err; 1647 } 1648 1649 /* If we've got too many, that implies a descriptor loop. */ 1650 if (num_bufs >= max) { 1651 virtio_error(vdev, "Looped descriptor"); 1652 goto err; 1653 } 1654 1655 /* loop over the indirect descriptor table */ 1656 len = address_space_cache_init(&indirect_desc_cache, 1657 vdev->dma_as, 1658 desc.addr, desc.len, false); 1659 desc_cache = &indirect_desc_cache; 1660 if (len < desc.len) { 1661 virtio_error(vdev, "Cannot map indirect buffer"); 1662 goto err; 1663 } 1664 1665 max = desc.len / sizeof(VRingPackedDesc); 1666 num_bufs = i = 0; 1667 vring_packed_desc_read(vdev, &desc, desc_cache, i, false); 1668 } 1669 1670 do { 1671 /* If we've got too many, that implies a descriptor loop. */ 1672 if (++num_bufs > max) { 1673 virtio_error(vdev, "Looped descriptor"); 1674 goto err; 1675 } 1676 1677 if (desc.flags & VRING_DESC_F_WRITE) { 1678 in_total += desc.len; 1679 } else { 1680 out_total += desc.len; 1681 } 1682 if (in_total >= max_in_bytes && out_total >= max_out_bytes) { 1683 goto done; 1684 } 1685 1686 rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max, 1687 &i, desc_cache == 1688 &indirect_desc_cache); 1689 } while (rc == VIRTQUEUE_READ_DESC_MORE); 1690 1691 if (desc_cache == &indirect_desc_cache) { 1692 address_space_cache_destroy(&indirect_desc_cache); 1693 total_bufs++; 1694 idx++; 1695 } else { 1696 idx += num_bufs - total_bufs; 1697 total_bufs = num_bufs; 1698 } 1699 1700 if (idx >= vq->vring.num) { 1701 idx -= vq->vring.num; 1702 wrap_counter ^= 1; 1703 } 1704 } 1705 1706 /* Record the index and wrap counter for a kick we want */ 1707 vq->shadow_avail_idx = idx; 1708 vq->shadow_avail_wrap_counter = wrap_counter; 1709 done: 1710 address_space_cache_destroy(&indirect_desc_cache); 1711 if (in_bytes) { 1712 *in_bytes = in_total; 1713 } 1714 if (out_bytes) { 1715 *out_bytes = out_total; 1716 } 1717 return; 1718 1719 err: 1720 in_total = out_total = 0; 1721 goto done; 1722 } 1723 1724 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes, 1725 unsigned int *out_bytes, 1726 unsigned max_in_bytes, unsigned max_out_bytes) 1727 { 1728 uint16_t desc_size; 1729 VRingMemoryRegionCaches *caches; 1730 1731 RCU_READ_LOCK_GUARD(); 1732 1733 if (unlikely(!vq->vring.desc)) { 1734 goto err; 1735 } 1736 1737 caches = vring_get_region_caches(vq); 1738 if (!caches) { 1739 goto err; 1740 } 1741 1742 desc_size = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ? 1743 sizeof(VRingPackedDesc) : sizeof(VRingDesc); 1744 if (caches->desc.len < vq->vring.num * desc_size) { 1745 virtio_error(vq->vdev, "Cannot map descriptor ring"); 1746 goto err; 1747 } 1748 1749 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1750 virtqueue_packed_get_avail_bytes(vq, in_bytes, out_bytes, 1751 max_in_bytes, max_out_bytes, 1752 caches); 1753 } else { 1754 virtqueue_split_get_avail_bytes(vq, in_bytes, out_bytes, 1755 max_in_bytes, max_out_bytes, 1756 caches); 1757 } 1758 1759 return; 1760 err: 1761 if (in_bytes) { 1762 *in_bytes = 0; 1763 } 1764 if (out_bytes) { 1765 *out_bytes = 0; 1766 } 1767 } 1768 1769 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes, 1770 unsigned int out_bytes) 1771 { 1772 unsigned int in_total, out_total; 1773 1774 virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes); 1775 return in_bytes <= in_total && out_bytes <= out_total; 1776 } 1777 1778 static bool virtqueue_map_desc(VirtIODevice *vdev, unsigned int *p_num_sg, 1779 hwaddr *addr, struct iovec *iov, 1780 unsigned int max_num_sg, bool is_write, 1781 hwaddr pa, size_t sz) 1782 { 1783 bool ok = false; 1784 unsigned num_sg = *p_num_sg; 1785 assert(num_sg <= max_num_sg); 1786 1787 if (!sz) { 1788 virtio_error(vdev, "virtio: zero sized buffers are not allowed"); 1789 goto out; 1790 } 1791 1792 while (sz) { 1793 hwaddr len = sz; 1794 1795 if (num_sg == max_num_sg) { 1796 virtio_error(vdev, "virtio: too many write descriptors in " 1797 "indirect table"); 1798 goto out; 1799 } 1800 1801 iov[num_sg].iov_base = dma_memory_map(vdev->dma_as, pa, &len, 1802 is_write ? 1803 DMA_DIRECTION_FROM_DEVICE : 1804 DMA_DIRECTION_TO_DEVICE, 1805 MEMTXATTRS_UNSPECIFIED); 1806 if (!iov[num_sg].iov_base) { 1807 virtio_error(vdev, "virtio: bogus descriptor or out of resources"); 1808 goto out; 1809 } 1810 1811 iov[num_sg].iov_len = len; 1812 addr[num_sg] = pa; 1813 1814 sz -= len; 1815 pa += len; 1816 num_sg++; 1817 } 1818 ok = true; 1819 1820 out: 1821 *p_num_sg = num_sg; 1822 return ok; 1823 } 1824 1825 /* Only used by error code paths before we have a VirtQueueElement (therefore 1826 * virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to 1827 * yet. 1828 */ 1829 static void virtqueue_undo_map_desc(unsigned int out_num, unsigned int in_num, 1830 struct iovec *iov) 1831 { 1832 unsigned int i; 1833 1834 for (i = 0; i < out_num + in_num; i++) { 1835 int is_write = i >= out_num; 1836 1837 cpu_physical_memory_unmap(iov->iov_base, iov->iov_len, is_write, 0); 1838 iov++; 1839 } 1840 } 1841 1842 static void virtqueue_map_iovec(VirtIODevice *vdev, struct iovec *sg, 1843 hwaddr *addr, unsigned int num_sg, 1844 bool is_write) 1845 { 1846 unsigned int i; 1847 hwaddr len; 1848 1849 for (i = 0; i < num_sg; i++) { 1850 len = sg[i].iov_len; 1851 sg[i].iov_base = dma_memory_map(vdev->dma_as, 1852 addr[i], &len, is_write ? 1853 DMA_DIRECTION_FROM_DEVICE : 1854 DMA_DIRECTION_TO_DEVICE, 1855 MEMTXATTRS_UNSPECIFIED); 1856 if (!sg[i].iov_base) { 1857 error_report("virtio: error trying to map MMIO memory"); 1858 exit(1); 1859 } 1860 if (len != sg[i].iov_len) { 1861 error_report("virtio: unexpected memory split"); 1862 exit(1); 1863 } 1864 } 1865 } 1866 1867 void virtqueue_map(VirtIODevice *vdev, VirtQueueElement *elem) 1868 { 1869 virtqueue_map_iovec(vdev, elem->in_sg, elem->in_addr, elem->in_num, true); 1870 virtqueue_map_iovec(vdev, elem->out_sg, elem->out_addr, elem->out_num, 1871 false); 1872 } 1873 1874 static void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num) 1875 { 1876 VirtQueueElement *elem; 1877 size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0])); 1878 size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]); 1879 size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]); 1880 size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0])); 1881 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]); 1882 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]); 1883 1884 assert(sz >= sizeof(VirtQueueElement)); 1885 elem = g_malloc(out_sg_end); 1886 trace_virtqueue_alloc_element(elem, sz, in_num, out_num); 1887 elem->out_num = out_num; 1888 elem->in_num = in_num; 1889 elem->in_addr = (void *)elem + in_addr_ofs; 1890 elem->out_addr = (void *)elem + out_addr_ofs; 1891 elem->in_sg = (void *)elem + in_sg_ofs; 1892 elem->out_sg = (void *)elem + out_sg_ofs; 1893 return elem; 1894 } 1895 1896 static void *virtqueue_split_pop(VirtQueue *vq, size_t sz) 1897 { 1898 unsigned int i, head, max; 1899 VRingMemoryRegionCaches *caches; 1900 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 1901 MemoryRegionCache *desc_cache; 1902 int64_t len; 1903 VirtIODevice *vdev = vq->vdev; 1904 VirtQueueElement *elem = NULL; 1905 unsigned out_num, in_num, elem_entries; 1906 hwaddr addr[VIRTQUEUE_MAX_SIZE]; 1907 struct iovec iov[VIRTQUEUE_MAX_SIZE]; 1908 VRingDesc desc; 1909 int rc; 1910 1911 RCU_READ_LOCK_GUARD(); 1912 if (virtio_queue_empty_rcu(vq)) { 1913 goto done; 1914 } 1915 /* Needed after virtio_queue_empty(), see comment in 1916 * virtqueue_num_heads(). */ 1917 smp_rmb(); 1918 1919 /* When we start there are none of either input nor output. */ 1920 out_num = in_num = elem_entries = 0; 1921 1922 max = vq->vring.num; 1923 1924 if (vq->inuse >= vq->vring.num) { 1925 virtio_error(vdev, "Virtqueue size exceeded"); 1926 goto done; 1927 } 1928 1929 if (!virtqueue_get_head(vq, vq->last_avail_idx++, &head)) { 1930 goto done; 1931 } 1932 1933 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 1934 vring_set_avail_event(vq, vq->last_avail_idx); 1935 } 1936 1937 i = head; 1938 1939 caches = vring_get_region_caches(vq); 1940 if (!caches) { 1941 virtio_error(vdev, "Region caches not initialized"); 1942 goto done; 1943 } 1944 1945 if (caches->desc.len < max * sizeof(VRingDesc)) { 1946 virtio_error(vdev, "Cannot map descriptor ring"); 1947 goto done; 1948 } 1949 1950 desc_cache = &caches->desc; 1951 vring_split_desc_read(vdev, &desc, desc_cache, i); 1952 if (desc.flags & VRING_DESC_F_INDIRECT) { 1953 if (!desc.len || (desc.len % sizeof(VRingDesc))) { 1954 virtio_error(vdev, "Invalid size for indirect buffer table"); 1955 goto done; 1956 } 1957 1958 /* loop over the indirect descriptor table */ 1959 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, 1960 desc.addr, desc.len, false); 1961 desc_cache = &indirect_desc_cache; 1962 if (len < desc.len) { 1963 virtio_error(vdev, "Cannot map indirect buffer"); 1964 goto done; 1965 } 1966 1967 max = desc.len / sizeof(VRingDesc); 1968 i = 0; 1969 vring_split_desc_read(vdev, &desc, desc_cache, i); 1970 } 1971 1972 /* Collect all the descriptors */ 1973 do { 1974 bool map_ok; 1975 1976 if (desc.flags & VRING_DESC_F_WRITE) { 1977 map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num, 1978 iov + out_num, 1979 VIRTQUEUE_MAX_SIZE - out_num, true, 1980 desc.addr, desc.len); 1981 } else { 1982 if (in_num) { 1983 virtio_error(vdev, "Incorrect order for descriptors"); 1984 goto err_undo_map; 1985 } 1986 map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov, 1987 VIRTQUEUE_MAX_SIZE, false, 1988 desc.addr, desc.len); 1989 } 1990 if (!map_ok) { 1991 goto err_undo_map; 1992 } 1993 1994 /* If we've got too many, that implies a descriptor loop. */ 1995 if (++elem_entries > max) { 1996 virtio_error(vdev, "Looped descriptor"); 1997 goto err_undo_map; 1998 } 1999 2000 rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max, &i); 2001 } while (rc == VIRTQUEUE_READ_DESC_MORE); 2002 2003 if (rc == VIRTQUEUE_READ_DESC_ERROR) { 2004 goto err_undo_map; 2005 } 2006 2007 /* Now copy what we have collected and mapped */ 2008 elem = virtqueue_alloc_element(sz, out_num, in_num); 2009 elem->index = head; 2010 elem->ndescs = 1; 2011 for (i = 0; i < out_num; i++) { 2012 elem->out_addr[i] = addr[i]; 2013 elem->out_sg[i] = iov[i]; 2014 } 2015 for (i = 0; i < in_num; i++) { 2016 elem->in_addr[i] = addr[out_num + i]; 2017 elem->in_sg[i] = iov[out_num + i]; 2018 } 2019 2020 vq->inuse++; 2021 2022 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); 2023 done: 2024 address_space_cache_destroy(&indirect_desc_cache); 2025 2026 return elem; 2027 2028 err_undo_map: 2029 virtqueue_undo_map_desc(out_num, in_num, iov); 2030 goto done; 2031 } 2032 2033 static void *virtqueue_packed_pop(VirtQueue *vq, size_t sz) 2034 { 2035 unsigned int i, max; 2036 VRingMemoryRegionCaches *caches; 2037 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 2038 MemoryRegionCache *desc_cache; 2039 int64_t len; 2040 VirtIODevice *vdev = vq->vdev; 2041 VirtQueueElement *elem = NULL; 2042 unsigned out_num, in_num, elem_entries; 2043 hwaddr addr[VIRTQUEUE_MAX_SIZE]; 2044 struct iovec iov[VIRTQUEUE_MAX_SIZE]; 2045 VRingPackedDesc desc; 2046 uint16_t id; 2047 int rc; 2048 2049 RCU_READ_LOCK_GUARD(); 2050 if (virtio_queue_packed_empty_rcu(vq)) { 2051 goto done; 2052 } 2053 2054 /* When we start there are none of either input nor output. */ 2055 out_num = in_num = elem_entries = 0; 2056 2057 max = vq->vring.num; 2058 2059 if (vq->inuse >= vq->vring.num) { 2060 virtio_error(vdev, "Virtqueue size exceeded"); 2061 goto done; 2062 } 2063 2064 i = vq->last_avail_idx; 2065 2066 caches = vring_get_region_caches(vq); 2067 if (!caches) { 2068 virtio_error(vdev, "Region caches not initialized"); 2069 goto done; 2070 } 2071 2072 if (caches->desc.len < max * sizeof(VRingDesc)) { 2073 virtio_error(vdev, "Cannot map descriptor ring"); 2074 goto done; 2075 } 2076 2077 desc_cache = &caches->desc; 2078 vring_packed_desc_read(vdev, &desc, desc_cache, i, true); 2079 id = desc.id; 2080 if (desc.flags & VRING_DESC_F_INDIRECT) { 2081 if (desc.len % sizeof(VRingPackedDesc)) { 2082 virtio_error(vdev, "Invalid size for indirect buffer table"); 2083 goto done; 2084 } 2085 2086 /* loop over the indirect descriptor table */ 2087 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, 2088 desc.addr, desc.len, false); 2089 desc_cache = &indirect_desc_cache; 2090 if (len < desc.len) { 2091 virtio_error(vdev, "Cannot map indirect buffer"); 2092 goto done; 2093 } 2094 2095 max = desc.len / sizeof(VRingPackedDesc); 2096 i = 0; 2097 vring_packed_desc_read(vdev, &desc, desc_cache, i, false); 2098 } 2099 2100 /* Collect all the descriptors */ 2101 do { 2102 bool map_ok; 2103 2104 if (desc.flags & VRING_DESC_F_WRITE) { 2105 map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num, 2106 iov + out_num, 2107 VIRTQUEUE_MAX_SIZE - out_num, true, 2108 desc.addr, desc.len); 2109 } else { 2110 if (in_num) { 2111 virtio_error(vdev, "Incorrect order for descriptors"); 2112 goto err_undo_map; 2113 } 2114 map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov, 2115 VIRTQUEUE_MAX_SIZE, false, 2116 desc.addr, desc.len); 2117 } 2118 if (!map_ok) { 2119 goto err_undo_map; 2120 } 2121 2122 /* If we've got too many, that implies a descriptor loop. */ 2123 if (++elem_entries > max) { 2124 virtio_error(vdev, "Looped descriptor"); 2125 goto err_undo_map; 2126 } 2127 2128 rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max, &i, 2129 desc_cache == 2130 &indirect_desc_cache); 2131 } while (rc == VIRTQUEUE_READ_DESC_MORE); 2132 2133 /* Now copy what we have collected and mapped */ 2134 elem = virtqueue_alloc_element(sz, out_num, in_num); 2135 for (i = 0; i < out_num; i++) { 2136 elem->out_addr[i] = addr[i]; 2137 elem->out_sg[i] = iov[i]; 2138 } 2139 for (i = 0; i < in_num; i++) { 2140 elem->in_addr[i] = addr[out_num + i]; 2141 elem->in_sg[i] = iov[out_num + i]; 2142 } 2143 2144 elem->index = id; 2145 elem->ndescs = (desc_cache == &indirect_desc_cache) ? 1 : elem_entries; 2146 vq->last_avail_idx += elem->ndescs; 2147 vq->inuse += elem->ndescs; 2148 2149 if (vq->last_avail_idx >= vq->vring.num) { 2150 vq->last_avail_idx -= vq->vring.num; 2151 vq->last_avail_wrap_counter ^= 1; 2152 } 2153 2154 vq->shadow_avail_idx = vq->last_avail_idx; 2155 vq->shadow_avail_wrap_counter = vq->last_avail_wrap_counter; 2156 2157 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); 2158 done: 2159 address_space_cache_destroy(&indirect_desc_cache); 2160 2161 return elem; 2162 2163 err_undo_map: 2164 virtqueue_undo_map_desc(out_num, in_num, iov); 2165 goto done; 2166 } 2167 2168 void *virtqueue_pop(VirtQueue *vq, size_t sz) 2169 { 2170 if (virtio_device_disabled(vq->vdev)) { 2171 return NULL; 2172 } 2173 2174 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 2175 return virtqueue_packed_pop(vq, sz); 2176 } else { 2177 return virtqueue_split_pop(vq, sz); 2178 } 2179 } 2180 2181 static unsigned int virtqueue_packed_drop_all(VirtQueue *vq) 2182 { 2183 VRingMemoryRegionCaches *caches; 2184 MemoryRegionCache *desc_cache; 2185 unsigned int dropped = 0; 2186 VirtQueueElement elem = {}; 2187 VirtIODevice *vdev = vq->vdev; 2188 VRingPackedDesc desc; 2189 2190 RCU_READ_LOCK_GUARD(); 2191 2192 caches = vring_get_region_caches(vq); 2193 if (!caches) { 2194 return 0; 2195 } 2196 2197 desc_cache = &caches->desc; 2198 2199 virtio_queue_set_notification(vq, 0); 2200 2201 while (vq->inuse < vq->vring.num) { 2202 unsigned int idx = vq->last_avail_idx; 2203 /* 2204 * works similar to virtqueue_pop but does not map buffers 2205 * and does not allocate any memory. 2206 */ 2207 vring_packed_desc_read(vdev, &desc, desc_cache, 2208 vq->last_avail_idx , true); 2209 if (!is_desc_avail(desc.flags, vq->last_avail_wrap_counter)) { 2210 break; 2211 } 2212 elem.index = desc.id; 2213 elem.ndescs = 1; 2214 while (virtqueue_packed_read_next_desc(vq, &desc, desc_cache, 2215 vq->vring.num, &idx, false)) { 2216 ++elem.ndescs; 2217 } 2218 /* 2219 * immediately push the element, nothing to unmap 2220 * as both in_num and out_num are set to 0. 2221 */ 2222 virtqueue_push(vq, &elem, 0); 2223 dropped++; 2224 vq->last_avail_idx += elem.ndescs; 2225 if (vq->last_avail_idx >= vq->vring.num) { 2226 vq->last_avail_idx -= vq->vring.num; 2227 vq->last_avail_wrap_counter ^= 1; 2228 } 2229 } 2230 2231 return dropped; 2232 } 2233 2234 static unsigned int virtqueue_split_drop_all(VirtQueue *vq) 2235 { 2236 unsigned int dropped = 0; 2237 VirtQueueElement elem = {}; 2238 VirtIODevice *vdev = vq->vdev; 2239 bool fEventIdx = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); 2240 2241 while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) { 2242 /* works similar to virtqueue_pop but does not map buffers 2243 * and does not allocate any memory */ 2244 smp_rmb(); 2245 if (!virtqueue_get_head(vq, vq->last_avail_idx, &elem.index)) { 2246 break; 2247 } 2248 vq->inuse++; 2249 vq->last_avail_idx++; 2250 if (fEventIdx) { 2251 vring_set_avail_event(vq, vq->last_avail_idx); 2252 } 2253 /* immediately push the element, nothing to unmap 2254 * as both in_num and out_num are set to 0 */ 2255 virtqueue_push(vq, &elem, 0); 2256 dropped++; 2257 } 2258 2259 return dropped; 2260 } 2261 2262 /* virtqueue_drop_all: 2263 * @vq: The #VirtQueue 2264 * Drops all queued buffers and indicates them to the guest 2265 * as if they are done. Useful when buffers can not be 2266 * processed but must be returned to the guest. 2267 */ 2268 unsigned int virtqueue_drop_all(VirtQueue *vq) 2269 { 2270 struct VirtIODevice *vdev = vq->vdev; 2271 2272 if (virtio_device_disabled(vq->vdev)) { 2273 return 0; 2274 } 2275 2276 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 2277 return virtqueue_packed_drop_all(vq); 2278 } else { 2279 return virtqueue_split_drop_all(vq); 2280 } 2281 } 2282 2283 /* Reading and writing a structure directly to QEMUFile is *awful*, but 2284 * it is what QEMU has always done by mistake. We can change it sooner 2285 * or later by bumping the version number of the affected vm states. 2286 * In the meanwhile, since the in-memory layout of VirtQueueElement 2287 * has changed, we need to marshal to and from the layout that was 2288 * used before the change. 2289 */ 2290 typedef struct VirtQueueElementOld { 2291 unsigned int index; 2292 unsigned int out_num; 2293 unsigned int in_num; 2294 hwaddr in_addr[VIRTQUEUE_MAX_SIZE]; 2295 hwaddr out_addr[VIRTQUEUE_MAX_SIZE]; 2296 struct iovec in_sg[VIRTQUEUE_MAX_SIZE]; 2297 struct iovec out_sg[VIRTQUEUE_MAX_SIZE]; 2298 } VirtQueueElementOld; 2299 2300 void *qemu_get_virtqueue_element(VirtIODevice *vdev, QEMUFile *f, size_t sz) 2301 { 2302 VirtQueueElement *elem; 2303 VirtQueueElementOld data; 2304 int i; 2305 2306 qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld)); 2307 2308 /* TODO: teach all callers that this can fail, and return failure instead 2309 * of asserting here. 2310 * This is just one thing (there are probably more) that must be 2311 * fixed before we can allow NDEBUG compilation. 2312 */ 2313 assert(ARRAY_SIZE(data.in_addr) >= data.in_num); 2314 assert(ARRAY_SIZE(data.out_addr) >= data.out_num); 2315 2316 elem = virtqueue_alloc_element(sz, data.out_num, data.in_num); 2317 elem->index = data.index; 2318 2319 for (i = 0; i < elem->in_num; i++) { 2320 elem->in_addr[i] = data.in_addr[i]; 2321 } 2322 2323 for (i = 0; i < elem->out_num; i++) { 2324 elem->out_addr[i] = data.out_addr[i]; 2325 } 2326 2327 for (i = 0; i < elem->in_num; i++) { 2328 /* Base is overwritten by virtqueue_map. */ 2329 elem->in_sg[i].iov_base = 0; 2330 elem->in_sg[i].iov_len = data.in_sg[i].iov_len; 2331 } 2332 2333 for (i = 0; i < elem->out_num; i++) { 2334 /* Base is overwritten by virtqueue_map. */ 2335 elem->out_sg[i].iov_base = 0; 2336 elem->out_sg[i].iov_len = data.out_sg[i].iov_len; 2337 } 2338 2339 if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 2340 qemu_get_be32s(f, &elem->ndescs); 2341 } 2342 2343 virtqueue_map(vdev, elem); 2344 return elem; 2345 } 2346 2347 void qemu_put_virtqueue_element(VirtIODevice *vdev, QEMUFile *f, 2348 VirtQueueElement *elem) 2349 { 2350 VirtQueueElementOld data; 2351 int i; 2352 2353 memset(&data, 0, sizeof(data)); 2354 data.index = elem->index; 2355 data.in_num = elem->in_num; 2356 data.out_num = elem->out_num; 2357 2358 for (i = 0; i < elem->in_num; i++) { 2359 data.in_addr[i] = elem->in_addr[i]; 2360 } 2361 2362 for (i = 0; i < elem->out_num; i++) { 2363 data.out_addr[i] = elem->out_addr[i]; 2364 } 2365 2366 for (i = 0; i < elem->in_num; i++) { 2367 /* Base is overwritten by virtqueue_map when loading. Do not 2368 * save it, as it would leak the QEMU address space layout. */ 2369 data.in_sg[i].iov_len = elem->in_sg[i].iov_len; 2370 } 2371 2372 for (i = 0; i < elem->out_num; i++) { 2373 /* Do not save iov_base as above. */ 2374 data.out_sg[i].iov_len = elem->out_sg[i].iov_len; 2375 } 2376 2377 if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 2378 qemu_put_be32s(f, &elem->ndescs); 2379 } 2380 2381 qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld)); 2382 } 2383 2384 /* virtio device */ 2385 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector) 2386 { 2387 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2388 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 2389 2390 if (virtio_device_disabled(vdev)) { 2391 return; 2392 } 2393 2394 if (k->notify) { 2395 k->notify(qbus->parent, vector); 2396 } 2397 } 2398 2399 void virtio_update_irq(VirtIODevice *vdev) 2400 { 2401 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); 2402 } 2403 2404 static int virtio_validate_features(VirtIODevice *vdev) 2405 { 2406 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2407 2408 if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM) && 2409 !virtio_vdev_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) { 2410 return -EFAULT; 2411 } 2412 2413 if (k->validate_features) { 2414 return k->validate_features(vdev); 2415 } else { 2416 return 0; 2417 } 2418 } 2419 2420 int virtio_set_status(VirtIODevice *vdev, uint8_t val) 2421 { 2422 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2423 trace_virtio_set_status(vdev, val); 2424 2425 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 2426 if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) && 2427 val & VIRTIO_CONFIG_S_FEATURES_OK) { 2428 int ret = virtio_validate_features(vdev); 2429 2430 if (ret) { 2431 return ret; 2432 } 2433 } 2434 } 2435 2436 if ((vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) != 2437 (val & VIRTIO_CONFIG_S_DRIVER_OK)) { 2438 virtio_set_started(vdev, val & VIRTIO_CONFIG_S_DRIVER_OK); 2439 } 2440 2441 if (k->set_status) { 2442 k->set_status(vdev, val); 2443 } 2444 vdev->status = val; 2445 2446 return 0; 2447 } 2448 2449 static enum virtio_device_endian virtio_default_endian(void) 2450 { 2451 if (target_words_bigendian()) { 2452 return VIRTIO_DEVICE_ENDIAN_BIG; 2453 } else { 2454 return VIRTIO_DEVICE_ENDIAN_LITTLE; 2455 } 2456 } 2457 2458 static enum virtio_device_endian virtio_current_cpu_endian(void) 2459 { 2460 if (cpu_virtio_is_big_endian(current_cpu)) { 2461 return VIRTIO_DEVICE_ENDIAN_BIG; 2462 } else { 2463 return VIRTIO_DEVICE_ENDIAN_LITTLE; 2464 } 2465 } 2466 2467 static void __virtio_queue_reset(VirtIODevice *vdev, uint32_t i) 2468 { 2469 vdev->vq[i].vring.desc = 0; 2470 vdev->vq[i].vring.avail = 0; 2471 vdev->vq[i].vring.used = 0; 2472 vdev->vq[i].last_avail_idx = 0; 2473 vdev->vq[i].shadow_avail_idx = 0; 2474 vdev->vq[i].used_idx = 0; 2475 vdev->vq[i].last_avail_wrap_counter = true; 2476 vdev->vq[i].shadow_avail_wrap_counter = true; 2477 vdev->vq[i].used_wrap_counter = true; 2478 virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR); 2479 vdev->vq[i].signalled_used = 0; 2480 vdev->vq[i].signalled_used_valid = false; 2481 vdev->vq[i].notification = true; 2482 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default; 2483 vdev->vq[i].inuse = 0; 2484 virtio_virtqueue_reset_region_cache(&vdev->vq[i]); 2485 } 2486 2487 void virtio_queue_reset(VirtIODevice *vdev, uint32_t queue_index) 2488 { 2489 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2490 2491 if (k->queue_reset) { 2492 k->queue_reset(vdev, queue_index); 2493 } 2494 2495 __virtio_queue_reset(vdev, queue_index); 2496 } 2497 2498 void virtio_queue_enable(VirtIODevice *vdev, uint32_t queue_index) 2499 { 2500 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2501 2502 if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 2503 error_report("queue_enable is only suppported in devices of virtio " 2504 "1.0 or later."); 2505 } 2506 2507 if (k->queue_enable) { 2508 k->queue_enable(vdev, queue_index); 2509 } 2510 } 2511 2512 void virtio_reset(void *opaque) 2513 { 2514 VirtIODevice *vdev = opaque; 2515 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2516 int i; 2517 2518 virtio_set_status(vdev, 0); 2519 if (current_cpu) { 2520 /* Guest initiated reset */ 2521 vdev->device_endian = virtio_current_cpu_endian(); 2522 } else { 2523 /* System reset */ 2524 vdev->device_endian = virtio_default_endian(); 2525 } 2526 2527 if (k->reset) { 2528 k->reset(vdev); 2529 } 2530 2531 vdev->start_on_kick = false; 2532 vdev->started = false; 2533 vdev->broken = false; 2534 vdev->guest_features = 0; 2535 vdev->queue_sel = 0; 2536 vdev->status = 0; 2537 vdev->disabled = false; 2538 qatomic_set(&vdev->isr, 0); 2539 vdev->config_vector = VIRTIO_NO_VECTOR; 2540 virtio_notify_vector(vdev, vdev->config_vector); 2541 2542 for(i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2543 __virtio_queue_reset(vdev, i); 2544 } 2545 } 2546 2547 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr) 2548 { 2549 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2550 uint8_t val; 2551 2552 if (addr + sizeof(val) > vdev->config_len) { 2553 return (uint32_t)-1; 2554 } 2555 2556 k->get_config(vdev, vdev->config); 2557 2558 val = ldub_p(vdev->config + addr); 2559 return val; 2560 } 2561 2562 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr) 2563 { 2564 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2565 uint16_t val; 2566 2567 if (addr + sizeof(val) > vdev->config_len) { 2568 return (uint32_t)-1; 2569 } 2570 2571 k->get_config(vdev, vdev->config); 2572 2573 val = lduw_p(vdev->config + addr); 2574 return val; 2575 } 2576 2577 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr) 2578 { 2579 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2580 uint32_t val; 2581 2582 if (addr + sizeof(val) > vdev->config_len) { 2583 return (uint32_t)-1; 2584 } 2585 2586 k->get_config(vdev, vdev->config); 2587 2588 val = ldl_p(vdev->config + addr); 2589 return val; 2590 } 2591 2592 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data) 2593 { 2594 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2595 uint8_t val = data; 2596 2597 if (addr + sizeof(val) > vdev->config_len) { 2598 return; 2599 } 2600 2601 stb_p(vdev->config + addr, val); 2602 2603 if (k->set_config) { 2604 k->set_config(vdev, vdev->config); 2605 } 2606 } 2607 2608 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data) 2609 { 2610 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2611 uint16_t val = data; 2612 2613 if (addr + sizeof(val) > vdev->config_len) { 2614 return; 2615 } 2616 2617 stw_p(vdev->config + addr, val); 2618 2619 if (k->set_config) { 2620 k->set_config(vdev, vdev->config); 2621 } 2622 } 2623 2624 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data) 2625 { 2626 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2627 uint32_t val = data; 2628 2629 if (addr + sizeof(val) > vdev->config_len) { 2630 return; 2631 } 2632 2633 stl_p(vdev->config + addr, val); 2634 2635 if (k->set_config) { 2636 k->set_config(vdev, vdev->config); 2637 } 2638 } 2639 2640 uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr) 2641 { 2642 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2643 uint8_t val; 2644 2645 if (addr + sizeof(val) > vdev->config_len) { 2646 return (uint32_t)-1; 2647 } 2648 2649 k->get_config(vdev, vdev->config); 2650 2651 val = ldub_p(vdev->config + addr); 2652 return val; 2653 } 2654 2655 uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr) 2656 { 2657 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2658 uint16_t val; 2659 2660 if (addr + sizeof(val) > vdev->config_len) { 2661 return (uint32_t)-1; 2662 } 2663 2664 k->get_config(vdev, vdev->config); 2665 2666 val = lduw_le_p(vdev->config + addr); 2667 return val; 2668 } 2669 2670 uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr) 2671 { 2672 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2673 uint32_t val; 2674 2675 if (addr + sizeof(val) > vdev->config_len) { 2676 return (uint32_t)-1; 2677 } 2678 2679 k->get_config(vdev, vdev->config); 2680 2681 val = ldl_le_p(vdev->config + addr); 2682 return val; 2683 } 2684 2685 void virtio_config_modern_writeb(VirtIODevice *vdev, 2686 uint32_t addr, uint32_t data) 2687 { 2688 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2689 uint8_t val = data; 2690 2691 if (addr + sizeof(val) > vdev->config_len) { 2692 return; 2693 } 2694 2695 stb_p(vdev->config + addr, val); 2696 2697 if (k->set_config) { 2698 k->set_config(vdev, vdev->config); 2699 } 2700 } 2701 2702 void virtio_config_modern_writew(VirtIODevice *vdev, 2703 uint32_t addr, uint32_t data) 2704 { 2705 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2706 uint16_t val = data; 2707 2708 if (addr + sizeof(val) > vdev->config_len) { 2709 return; 2710 } 2711 2712 stw_le_p(vdev->config + addr, val); 2713 2714 if (k->set_config) { 2715 k->set_config(vdev, vdev->config); 2716 } 2717 } 2718 2719 void virtio_config_modern_writel(VirtIODevice *vdev, 2720 uint32_t addr, uint32_t data) 2721 { 2722 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2723 uint32_t val = data; 2724 2725 if (addr + sizeof(val) > vdev->config_len) { 2726 return; 2727 } 2728 2729 stl_le_p(vdev->config + addr, val); 2730 2731 if (k->set_config) { 2732 k->set_config(vdev, vdev->config); 2733 } 2734 } 2735 2736 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr) 2737 { 2738 if (!vdev->vq[n].vring.num) { 2739 return; 2740 } 2741 vdev->vq[n].vring.desc = addr; 2742 virtio_queue_update_rings(vdev, n); 2743 } 2744 2745 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n) 2746 { 2747 return vdev->vq[n].vring.desc; 2748 } 2749 2750 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc, 2751 hwaddr avail, hwaddr used) 2752 { 2753 if (!vdev->vq[n].vring.num) { 2754 return; 2755 } 2756 vdev->vq[n].vring.desc = desc; 2757 vdev->vq[n].vring.avail = avail; 2758 vdev->vq[n].vring.used = used; 2759 virtio_init_region_cache(vdev, n); 2760 } 2761 2762 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num) 2763 { 2764 /* Don't allow guest to flip queue between existent and 2765 * nonexistent states, or to set it to an invalid size. 2766 */ 2767 if (!!num != !!vdev->vq[n].vring.num || 2768 num > VIRTQUEUE_MAX_SIZE || 2769 num < 0) { 2770 return; 2771 } 2772 vdev->vq[n].vring.num = num; 2773 } 2774 2775 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector) 2776 { 2777 return QLIST_FIRST(&vdev->vector_queues[vector]); 2778 } 2779 2780 VirtQueue *virtio_vector_next_queue(VirtQueue *vq) 2781 { 2782 return QLIST_NEXT(vq, node); 2783 } 2784 2785 int virtio_queue_get_num(VirtIODevice *vdev, int n) 2786 { 2787 return vdev->vq[n].vring.num; 2788 } 2789 2790 int virtio_queue_get_max_num(VirtIODevice *vdev, int n) 2791 { 2792 return vdev->vq[n].vring.num_default; 2793 } 2794 2795 int virtio_get_num_queues(VirtIODevice *vdev) 2796 { 2797 int i; 2798 2799 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2800 if (!virtio_queue_get_num(vdev, i)) { 2801 break; 2802 } 2803 } 2804 2805 return i; 2806 } 2807 2808 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align) 2809 { 2810 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2811 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 2812 2813 /* virtio-1 compliant devices cannot change the alignment */ 2814 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 2815 error_report("tried to modify queue alignment for virtio-1 device"); 2816 return; 2817 } 2818 /* Check that the transport told us it was going to do this 2819 * (so a buggy transport will immediately assert rather than 2820 * silently failing to migrate this state) 2821 */ 2822 assert(k->has_variable_vring_alignment); 2823 2824 if (align) { 2825 vdev->vq[n].vring.align = align; 2826 virtio_queue_update_rings(vdev, n); 2827 } 2828 } 2829 2830 static void virtio_queue_notify_vq(VirtQueue *vq) 2831 { 2832 if (vq->vring.desc && vq->handle_output) { 2833 VirtIODevice *vdev = vq->vdev; 2834 2835 if (unlikely(vdev->broken)) { 2836 return; 2837 } 2838 2839 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); 2840 vq->handle_output(vdev, vq); 2841 2842 if (unlikely(vdev->start_on_kick)) { 2843 virtio_set_started(vdev, true); 2844 } 2845 } 2846 } 2847 2848 void virtio_queue_notify(VirtIODevice *vdev, int n) 2849 { 2850 VirtQueue *vq = &vdev->vq[n]; 2851 2852 if (unlikely(!vq->vring.desc || vdev->broken)) { 2853 return; 2854 } 2855 2856 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); 2857 if (vq->host_notifier_enabled) { 2858 event_notifier_set(&vq->host_notifier); 2859 } else if (vq->handle_output) { 2860 vq->handle_output(vdev, vq); 2861 2862 if (unlikely(vdev->start_on_kick)) { 2863 virtio_set_started(vdev, true); 2864 } 2865 } 2866 } 2867 2868 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n) 2869 { 2870 return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector : 2871 VIRTIO_NO_VECTOR; 2872 } 2873 2874 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector) 2875 { 2876 VirtQueue *vq = &vdev->vq[n]; 2877 2878 if (n < VIRTIO_QUEUE_MAX) { 2879 if (vdev->vector_queues && 2880 vdev->vq[n].vector != VIRTIO_NO_VECTOR) { 2881 QLIST_REMOVE(vq, node); 2882 } 2883 vdev->vq[n].vector = vector; 2884 if (vdev->vector_queues && 2885 vector != VIRTIO_NO_VECTOR) { 2886 QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node); 2887 } 2888 } 2889 } 2890 2891 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size, 2892 VirtIOHandleOutput handle_output) 2893 { 2894 int i; 2895 2896 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2897 if (vdev->vq[i].vring.num == 0) 2898 break; 2899 } 2900 2901 if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE) 2902 abort(); 2903 2904 vdev->vq[i].vring.num = queue_size; 2905 vdev->vq[i].vring.num_default = queue_size; 2906 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN; 2907 vdev->vq[i].handle_output = handle_output; 2908 vdev->vq[i].used_elems = g_new0(VirtQueueElement, queue_size); 2909 2910 return &vdev->vq[i]; 2911 } 2912 2913 void virtio_delete_queue(VirtQueue *vq) 2914 { 2915 vq->vring.num = 0; 2916 vq->vring.num_default = 0; 2917 vq->handle_output = NULL; 2918 g_free(vq->used_elems); 2919 vq->used_elems = NULL; 2920 virtio_virtqueue_reset_region_cache(vq); 2921 } 2922 2923 void virtio_del_queue(VirtIODevice *vdev, int n) 2924 { 2925 if (n < 0 || n >= VIRTIO_QUEUE_MAX) { 2926 abort(); 2927 } 2928 2929 virtio_delete_queue(&vdev->vq[n]); 2930 } 2931 2932 static void virtio_set_isr(VirtIODevice *vdev, int value) 2933 { 2934 uint8_t old = qatomic_read(&vdev->isr); 2935 2936 /* Do not write ISR if it does not change, so that its cacheline remains 2937 * shared in the common case where the guest does not read it. 2938 */ 2939 if ((old & value) != value) { 2940 qatomic_or(&vdev->isr, value); 2941 } 2942 } 2943 2944 /* Called within rcu_read_lock(). */ 2945 static bool virtio_split_should_notify(VirtIODevice *vdev, VirtQueue *vq) 2946 { 2947 uint16_t old, new; 2948 bool v; 2949 /* We need to expose used array entries before checking used event. */ 2950 smp_mb(); 2951 /* Always notify when queue is empty (when feature acknowledge) */ 2952 if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) && 2953 !vq->inuse && virtio_queue_empty(vq)) { 2954 return true; 2955 } 2956 2957 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 2958 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT); 2959 } 2960 2961 v = vq->signalled_used_valid; 2962 vq->signalled_used_valid = true; 2963 old = vq->signalled_used; 2964 new = vq->signalled_used = vq->used_idx; 2965 return !v || vring_need_event(vring_get_used_event(vq), new, old); 2966 } 2967 2968 static bool vring_packed_need_event(VirtQueue *vq, bool wrap, 2969 uint16_t off_wrap, uint16_t new, 2970 uint16_t old) 2971 { 2972 int off = off_wrap & ~(1 << 15); 2973 2974 if (wrap != off_wrap >> 15) { 2975 off -= vq->vring.num; 2976 } 2977 2978 return vring_need_event(off, new, old); 2979 } 2980 2981 /* Called within rcu_read_lock(). */ 2982 static bool virtio_packed_should_notify(VirtIODevice *vdev, VirtQueue *vq) 2983 { 2984 VRingPackedDescEvent e; 2985 uint16_t old, new; 2986 bool v; 2987 VRingMemoryRegionCaches *caches; 2988 2989 caches = vring_get_region_caches(vq); 2990 if (!caches) { 2991 return false; 2992 } 2993 2994 vring_packed_event_read(vdev, &caches->avail, &e); 2995 2996 old = vq->signalled_used; 2997 new = vq->signalled_used = vq->used_idx; 2998 v = vq->signalled_used_valid; 2999 vq->signalled_used_valid = true; 3000 3001 if (e.flags == VRING_PACKED_EVENT_FLAG_DISABLE) { 3002 return false; 3003 } else if (e.flags == VRING_PACKED_EVENT_FLAG_ENABLE) { 3004 return true; 3005 } 3006 3007 return !v || vring_packed_need_event(vq, vq->used_wrap_counter, 3008 e.off_wrap, new, old); 3009 } 3010 3011 /* Called within rcu_read_lock(). */ 3012 static bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq) 3013 { 3014 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3015 return virtio_packed_should_notify(vdev, vq); 3016 } else { 3017 return virtio_split_should_notify(vdev, vq); 3018 } 3019 } 3020 3021 void virtio_notify_irqfd(VirtIODevice *vdev, VirtQueue *vq) 3022 { 3023 WITH_RCU_READ_LOCK_GUARD() { 3024 if (!virtio_should_notify(vdev, vq)) { 3025 return; 3026 } 3027 } 3028 3029 trace_virtio_notify_irqfd(vdev, vq); 3030 3031 /* 3032 * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but 3033 * windows drivers included in virtio-win 1.8.0 (circa 2015) are 3034 * incorrectly polling this bit during crashdump and hibernation 3035 * in MSI mode, causing a hang if this bit is never updated. 3036 * Recent releases of Windows do not really shut down, but rather 3037 * log out and hibernate to make the next startup faster. Hence, 3038 * this manifested as a more serious hang during shutdown with 3039 * 3040 * Next driver release from 2016 fixed this problem, so working around it 3041 * is not a must, but it's easy to do so let's do it here. 3042 * 3043 * Note: it's safe to update ISR from any thread as it was switched 3044 * to an atomic operation. 3045 */ 3046 virtio_set_isr(vq->vdev, 0x1); 3047 event_notifier_set(&vq->guest_notifier); 3048 } 3049 3050 static void virtio_irq(VirtQueue *vq) 3051 { 3052 virtio_set_isr(vq->vdev, 0x1); 3053 virtio_notify_vector(vq->vdev, vq->vector); 3054 } 3055 3056 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq) 3057 { 3058 WITH_RCU_READ_LOCK_GUARD() { 3059 if (!virtio_should_notify(vdev, vq)) { 3060 return; 3061 } 3062 } 3063 3064 trace_virtio_notify(vdev, vq); 3065 virtio_irq(vq); 3066 } 3067 3068 void virtio_notify_config(VirtIODevice *vdev) 3069 { 3070 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 3071 return; 3072 3073 virtio_set_isr(vdev, 0x3); 3074 vdev->generation++; 3075 virtio_notify_vector(vdev, vdev->config_vector); 3076 } 3077 3078 static bool virtio_device_endian_needed(void *opaque) 3079 { 3080 VirtIODevice *vdev = opaque; 3081 3082 assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN); 3083 if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 3084 return vdev->device_endian != virtio_default_endian(); 3085 } 3086 /* Devices conforming to VIRTIO 1.0 or later are always LE. */ 3087 return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE; 3088 } 3089 3090 static bool virtio_64bit_features_needed(void *opaque) 3091 { 3092 VirtIODevice *vdev = opaque; 3093 3094 return (vdev->host_features >> 32) != 0; 3095 } 3096 3097 static bool virtio_virtqueue_needed(void *opaque) 3098 { 3099 VirtIODevice *vdev = opaque; 3100 3101 return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1); 3102 } 3103 3104 static bool virtio_packed_virtqueue_needed(void *opaque) 3105 { 3106 VirtIODevice *vdev = opaque; 3107 3108 return virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED); 3109 } 3110 3111 static bool virtio_ringsize_needed(void *opaque) 3112 { 3113 VirtIODevice *vdev = opaque; 3114 int i; 3115 3116 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 3117 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) { 3118 return true; 3119 } 3120 } 3121 return false; 3122 } 3123 3124 static bool virtio_extra_state_needed(void *opaque) 3125 { 3126 VirtIODevice *vdev = opaque; 3127 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3128 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3129 3130 return k->has_extra_state && 3131 k->has_extra_state(qbus->parent); 3132 } 3133 3134 static bool virtio_broken_needed(void *opaque) 3135 { 3136 VirtIODevice *vdev = opaque; 3137 3138 return vdev->broken; 3139 } 3140 3141 static bool virtio_started_needed(void *opaque) 3142 { 3143 VirtIODevice *vdev = opaque; 3144 3145 return vdev->started; 3146 } 3147 3148 static bool virtio_disabled_needed(void *opaque) 3149 { 3150 VirtIODevice *vdev = opaque; 3151 3152 return vdev->disabled; 3153 } 3154 3155 static const VMStateDescription vmstate_virtqueue = { 3156 .name = "virtqueue_state", 3157 .version_id = 1, 3158 .minimum_version_id = 1, 3159 .fields = (VMStateField[]) { 3160 VMSTATE_UINT64(vring.avail, struct VirtQueue), 3161 VMSTATE_UINT64(vring.used, struct VirtQueue), 3162 VMSTATE_END_OF_LIST() 3163 } 3164 }; 3165 3166 static const VMStateDescription vmstate_packed_virtqueue = { 3167 .name = "packed_virtqueue_state", 3168 .version_id = 1, 3169 .minimum_version_id = 1, 3170 .fields = (VMStateField[]) { 3171 VMSTATE_UINT16(last_avail_idx, struct VirtQueue), 3172 VMSTATE_BOOL(last_avail_wrap_counter, struct VirtQueue), 3173 VMSTATE_UINT16(used_idx, struct VirtQueue), 3174 VMSTATE_BOOL(used_wrap_counter, struct VirtQueue), 3175 VMSTATE_UINT32(inuse, struct VirtQueue), 3176 VMSTATE_END_OF_LIST() 3177 } 3178 }; 3179 3180 static const VMStateDescription vmstate_virtio_virtqueues = { 3181 .name = "virtio/virtqueues", 3182 .version_id = 1, 3183 .minimum_version_id = 1, 3184 .needed = &virtio_virtqueue_needed, 3185 .fields = (VMStateField[]) { 3186 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, 3187 VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue), 3188 VMSTATE_END_OF_LIST() 3189 } 3190 }; 3191 3192 static const VMStateDescription vmstate_virtio_packed_virtqueues = { 3193 .name = "virtio/packed_virtqueues", 3194 .version_id = 1, 3195 .minimum_version_id = 1, 3196 .needed = &virtio_packed_virtqueue_needed, 3197 .fields = (VMStateField[]) { 3198 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, 3199 VIRTIO_QUEUE_MAX, 0, vmstate_packed_virtqueue, VirtQueue), 3200 VMSTATE_END_OF_LIST() 3201 } 3202 }; 3203 3204 static const VMStateDescription vmstate_ringsize = { 3205 .name = "ringsize_state", 3206 .version_id = 1, 3207 .minimum_version_id = 1, 3208 .fields = (VMStateField[]) { 3209 VMSTATE_UINT32(vring.num_default, struct VirtQueue), 3210 VMSTATE_END_OF_LIST() 3211 } 3212 }; 3213 3214 static const VMStateDescription vmstate_virtio_ringsize = { 3215 .name = "virtio/ringsize", 3216 .version_id = 1, 3217 .minimum_version_id = 1, 3218 .needed = &virtio_ringsize_needed, 3219 .fields = (VMStateField[]) { 3220 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, 3221 VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue), 3222 VMSTATE_END_OF_LIST() 3223 } 3224 }; 3225 3226 static int get_extra_state(QEMUFile *f, void *pv, size_t size, 3227 const VMStateField *field) 3228 { 3229 VirtIODevice *vdev = pv; 3230 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3231 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3232 3233 if (!k->load_extra_state) { 3234 return -1; 3235 } else { 3236 return k->load_extra_state(qbus->parent, f); 3237 } 3238 } 3239 3240 static int put_extra_state(QEMUFile *f, void *pv, size_t size, 3241 const VMStateField *field, JSONWriter *vmdesc) 3242 { 3243 VirtIODevice *vdev = pv; 3244 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3245 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3246 3247 k->save_extra_state(qbus->parent, f); 3248 return 0; 3249 } 3250 3251 static const VMStateInfo vmstate_info_extra_state = { 3252 .name = "virtqueue_extra_state", 3253 .get = get_extra_state, 3254 .put = put_extra_state, 3255 }; 3256 3257 static const VMStateDescription vmstate_virtio_extra_state = { 3258 .name = "virtio/extra_state", 3259 .version_id = 1, 3260 .minimum_version_id = 1, 3261 .needed = &virtio_extra_state_needed, 3262 .fields = (VMStateField[]) { 3263 { 3264 .name = "extra_state", 3265 .version_id = 0, 3266 .field_exists = NULL, 3267 .size = 0, 3268 .info = &vmstate_info_extra_state, 3269 .flags = VMS_SINGLE, 3270 .offset = 0, 3271 }, 3272 VMSTATE_END_OF_LIST() 3273 } 3274 }; 3275 3276 static const VMStateDescription vmstate_virtio_device_endian = { 3277 .name = "virtio/device_endian", 3278 .version_id = 1, 3279 .minimum_version_id = 1, 3280 .needed = &virtio_device_endian_needed, 3281 .fields = (VMStateField[]) { 3282 VMSTATE_UINT8(device_endian, VirtIODevice), 3283 VMSTATE_END_OF_LIST() 3284 } 3285 }; 3286 3287 static const VMStateDescription vmstate_virtio_64bit_features = { 3288 .name = "virtio/64bit_features", 3289 .version_id = 1, 3290 .minimum_version_id = 1, 3291 .needed = &virtio_64bit_features_needed, 3292 .fields = (VMStateField[]) { 3293 VMSTATE_UINT64(guest_features, VirtIODevice), 3294 VMSTATE_END_OF_LIST() 3295 } 3296 }; 3297 3298 static const VMStateDescription vmstate_virtio_broken = { 3299 .name = "virtio/broken", 3300 .version_id = 1, 3301 .minimum_version_id = 1, 3302 .needed = &virtio_broken_needed, 3303 .fields = (VMStateField[]) { 3304 VMSTATE_BOOL(broken, VirtIODevice), 3305 VMSTATE_END_OF_LIST() 3306 } 3307 }; 3308 3309 static const VMStateDescription vmstate_virtio_started = { 3310 .name = "virtio/started", 3311 .version_id = 1, 3312 .minimum_version_id = 1, 3313 .needed = &virtio_started_needed, 3314 .fields = (VMStateField[]) { 3315 VMSTATE_BOOL(started, VirtIODevice), 3316 VMSTATE_END_OF_LIST() 3317 } 3318 }; 3319 3320 static const VMStateDescription vmstate_virtio_disabled = { 3321 .name = "virtio/disabled", 3322 .version_id = 1, 3323 .minimum_version_id = 1, 3324 .needed = &virtio_disabled_needed, 3325 .fields = (VMStateField[]) { 3326 VMSTATE_BOOL(disabled, VirtIODevice), 3327 VMSTATE_END_OF_LIST() 3328 } 3329 }; 3330 3331 static const VMStateDescription vmstate_virtio = { 3332 .name = "virtio", 3333 .version_id = 1, 3334 .minimum_version_id = 1, 3335 .fields = (VMStateField[]) { 3336 VMSTATE_END_OF_LIST() 3337 }, 3338 .subsections = (const VMStateDescription*[]) { 3339 &vmstate_virtio_device_endian, 3340 &vmstate_virtio_64bit_features, 3341 &vmstate_virtio_virtqueues, 3342 &vmstate_virtio_ringsize, 3343 &vmstate_virtio_broken, 3344 &vmstate_virtio_extra_state, 3345 &vmstate_virtio_started, 3346 &vmstate_virtio_packed_virtqueues, 3347 &vmstate_virtio_disabled, 3348 NULL 3349 } 3350 }; 3351 3352 int virtio_save(VirtIODevice *vdev, QEMUFile *f) 3353 { 3354 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3355 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3356 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 3357 uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff); 3358 int i; 3359 3360 if (k->save_config) { 3361 k->save_config(qbus->parent, f); 3362 } 3363 3364 qemu_put_8s(f, &vdev->status); 3365 qemu_put_8s(f, &vdev->isr); 3366 qemu_put_be16s(f, &vdev->queue_sel); 3367 qemu_put_be32s(f, &guest_features_lo); 3368 qemu_put_be32(f, vdev->config_len); 3369 qemu_put_buffer(f, vdev->config, vdev->config_len); 3370 3371 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 3372 if (vdev->vq[i].vring.num == 0) 3373 break; 3374 } 3375 3376 qemu_put_be32(f, i); 3377 3378 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 3379 if (vdev->vq[i].vring.num == 0) 3380 break; 3381 3382 qemu_put_be32(f, vdev->vq[i].vring.num); 3383 if (k->has_variable_vring_alignment) { 3384 qemu_put_be32(f, vdev->vq[i].vring.align); 3385 } 3386 /* 3387 * Save desc now, the rest of the ring addresses are saved in 3388 * subsections for VIRTIO-1 devices. 3389 */ 3390 qemu_put_be64(f, vdev->vq[i].vring.desc); 3391 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx); 3392 if (k->save_queue) { 3393 k->save_queue(qbus->parent, i, f); 3394 } 3395 } 3396 3397 if (vdc->save != NULL) { 3398 vdc->save(vdev, f); 3399 } 3400 3401 if (vdc->vmsd) { 3402 int ret = vmstate_save_state(f, vdc->vmsd, vdev, NULL); 3403 if (ret) { 3404 return ret; 3405 } 3406 } 3407 3408 /* Subsections */ 3409 return vmstate_save_state(f, &vmstate_virtio, vdev, NULL); 3410 } 3411 3412 /* A wrapper for use as a VMState .put function */ 3413 static int virtio_device_put(QEMUFile *f, void *opaque, size_t size, 3414 const VMStateField *field, JSONWriter *vmdesc) 3415 { 3416 return virtio_save(VIRTIO_DEVICE(opaque), f); 3417 } 3418 3419 /* A wrapper for use as a VMState .get function */ 3420 static int virtio_device_get(QEMUFile *f, void *opaque, size_t size, 3421 const VMStateField *field) 3422 { 3423 VirtIODevice *vdev = VIRTIO_DEVICE(opaque); 3424 DeviceClass *dc = DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev)); 3425 3426 return virtio_load(vdev, f, dc->vmsd->version_id); 3427 } 3428 3429 const VMStateInfo virtio_vmstate_info = { 3430 .name = "virtio", 3431 .get = virtio_device_get, 3432 .put = virtio_device_put, 3433 }; 3434 3435 static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val) 3436 { 3437 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 3438 bool bad = (val & ~(vdev->host_features)) != 0; 3439 3440 val &= vdev->host_features; 3441 if (k->set_features) { 3442 k->set_features(vdev, val); 3443 } 3444 vdev->guest_features = val; 3445 return bad ? -1 : 0; 3446 } 3447 3448 int virtio_set_features(VirtIODevice *vdev, uint64_t val) 3449 { 3450 int ret; 3451 /* 3452 * The driver must not attempt to set features after feature negotiation 3453 * has finished. 3454 */ 3455 if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) { 3456 return -EINVAL; 3457 } 3458 3459 if (val & (1ull << VIRTIO_F_BAD_FEATURE)) { 3460 qemu_log_mask(LOG_GUEST_ERROR, 3461 "%s: guest driver for %s has enabled UNUSED(30) feature bit!\n", 3462 __func__, vdev->name); 3463 } 3464 3465 ret = virtio_set_features_nocheck(vdev, val); 3466 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 3467 /* VIRTIO_RING_F_EVENT_IDX changes the size of the caches. */ 3468 int i; 3469 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 3470 if (vdev->vq[i].vring.num != 0) { 3471 virtio_init_region_cache(vdev, i); 3472 } 3473 } 3474 } 3475 if (!ret) { 3476 if (!virtio_device_started(vdev, vdev->status) && 3477 !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 3478 vdev->start_on_kick = true; 3479 } 3480 } 3481 return ret; 3482 } 3483 3484 size_t virtio_get_config_size(const VirtIOConfigSizeParams *params, 3485 uint64_t host_features) 3486 { 3487 size_t config_size = params->min_size; 3488 const VirtIOFeature *feature_sizes = params->feature_sizes; 3489 size_t i; 3490 3491 for (i = 0; feature_sizes[i].flags != 0; i++) { 3492 if (host_features & feature_sizes[i].flags) { 3493 config_size = MAX(feature_sizes[i].end, config_size); 3494 } 3495 } 3496 3497 assert(config_size <= params->max_size); 3498 return config_size; 3499 } 3500 3501 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id) 3502 { 3503 int i, ret; 3504 int32_t config_len; 3505 uint32_t num; 3506 uint32_t features; 3507 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3508 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3509 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 3510 3511 /* 3512 * We poison the endianness to ensure it does not get used before 3513 * subsections have been loaded. 3514 */ 3515 vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN; 3516 3517 if (k->load_config) { 3518 ret = k->load_config(qbus->parent, f); 3519 if (ret) 3520 return ret; 3521 } 3522 3523 qemu_get_8s(f, &vdev->status); 3524 qemu_get_8s(f, &vdev->isr); 3525 qemu_get_be16s(f, &vdev->queue_sel); 3526 if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) { 3527 return -1; 3528 } 3529 qemu_get_be32s(f, &features); 3530 3531 /* 3532 * Temporarily set guest_features low bits - needed by 3533 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS 3534 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ. 3535 * 3536 * Note: devices should always test host features in future - don't create 3537 * new dependencies like this. 3538 */ 3539 vdev->guest_features = features; 3540 3541 config_len = qemu_get_be32(f); 3542 3543 /* 3544 * There are cases where the incoming config can be bigger or smaller 3545 * than what we have; so load what we have space for, and skip 3546 * any excess that's in the stream. 3547 */ 3548 qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len)); 3549 3550 while (config_len > vdev->config_len) { 3551 qemu_get_byte(f); 3552 config_len--; 3553 } 3554 3555 num = qemu_get_be32(f); 3556 3557 if (num > VIRTIO_QUEUE_MAX) { 3558 error_report("Invalid number of virtqueues: 0x%x", num); 3559 return -1; 3560 } 3561 3562 for (i = 0; i < num; i++) { 3563 vdev->vq[i].vring.num = qemu_get_be32(f); 3564 if (k->has_variable_vring_alignment) { 3565 vdev->vq[i].vring.align = qemu_get_be32(f); 3566 } 3567 vdev->vq[i].vring.desc = qemu_get_be64(f); 3568 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx); 3569 vdev->vq[i].signalled_used_valid = false; 3570 vdev->vq[i].notification = true; 3571 3572 if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) { 3573 error_report("VQ %d address 0x0 " 3574 "inconsistent with Host index 0x%x", 3575 i, vdev->vq[i].last_avail_idx); 3576 return -1; 3577 } 3578 if (k->load_queue) { 3579 ret = k->load_queue(qbus->parent, i, f); 3580 if (ret) 3581 return ret; 3582 } 3583 } 3584 3585 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); 3586 3587 if (vdc->load != NULL) { 3588 ret = vdc->load(vdev, f, version_id); 3589 if (ret) { 3590 return ret; 3591 } 3592 } 3593 3594 if (vdc->vmsd) { 3595 ret = vmstate_load_state(f, vdc->vmsd, vdev, version_id); 3596 if (ret) { 3597 return ret; 3598 } 3599 } 3600 3601 /* Subsections */ 3602 ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1); 3603 if (ret) { 3604 return ret; 3605 } 3606 3607 if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) { 3608 vdev->device_endian = virtio_default_endian(); 3609 } 3610 3611 if (virtio_64bit_features_needed(vdev)) { 3612 /* 3613 * Subsection load filled vdev->guest_features. Run them 3614 * through virtio_set_features to sanity-check them against 3615 * host_features. 3616 */ 3617 uint64_t features64 = vdev->guest_features; 3618 if (virtio_set_features_nocheck(vdev, features64) < 0) { 3619 error_report("Features 0x%" PRIx64 " unsupported. " 3620 "Allowed features: 0x%" PRIx64, 3621 features64, vdev->host_features); 3622 return -1; 3623 } 3624 } else { 3625 if (virtio_set_features_nocheck(vdev, features) < 0) { 3626 error_report("Features 0x%x unsupported. " 3627 "Allowed features: 0x%" PRIx64, 3628 features, vdev->host_features); 3629 return -1; 3630 } 3631 } 3632 3633 if (!virtio_device_started(vdev, vdev->status) && 3634 !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 3635 vdev->start_on_kick = true; 3636 } 3637 3638 RCU_READ_LOCK_GUARD(); 3639 for (i = 0; i < num; i++) { 3640 if (vdev->vq[i].vring.desc) { 3641 uint16_t nheads; 3642 3643 /* 3644 * VIRTIO-1 devices migrate desc, used, and avail ring addresses so 3645 * only the region cache needs to be set up. Legacy devices need 3646 * to calculate used and avail ring addresses based on the desc 3647 * address. 3648 */ 3649 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 3650 virtio_init_region_cache(vdev, i); 3651 } else { 3652 virtio_queue_update_rings(vdev, i); 3653 } 3654 3655 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3656 vdev->vq[i].shadow_avail_idx = vdev->vq[i].last_avail_idx; 3657 vdev->vq[i].shadow_avail_wrap_counter = 3658 vdev->vq[i].last_avail_wrap_counter; 3659 continue; 3660 } 3661 3662 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx; 3663 /* Check it isn't doing strange things with descriptor numbers. */ 3664 if (nheads > vdev->vq[i].vring.num) { 3665 virtio_error(vdev, "VQ %d size 0x%x Guest index 0x%x " 3666 "inconsistent with Host index 0x%x: delta 0x%x", 3667 i, vdev->vq[i].vring.num, 3668 vring_avail_idx(&vdev->vq[i]), 3669 vdev->vq[i].last_avail_idx, nheads); 3670 vdev->vq[i].used_idx = 0; 3671 vdev->vq[i].shadow_avail_idx = 0; 3672 vdev->vq[i].inuse = 0; 3673 continue; 3674 } 3675 vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]); 3676 vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]); 3677 3678 /* 3679 * Some devices migrate VirtQueueElements that have been popped 3680 * from the avail ring but not yet returned to the used ring. 3681 * Since max ring size < UINT16_MAX it's safe to use modulo 3682 * UINT16_MAX + 1 subtraction. 3683 */ 3684 vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx - 3685 vdev->vq[i].used_idx); 3686 if (vdev->vq[i].inuse > vdev->vq[i].vring.num) { 3687 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - " 3688 "used_idx 0x%x", 3689 i, vdev->vq[i].vring.num, 3690 vdev->vq[i].last_avail_idx, 3691 vdev->vq[i].used_idx); 3692 return -1; 3693 } 3694 } 3695 } 3696 3697 if (vdc->post_load) { 3698 ret = vdc->post_load(vdev); 3699 if (ret) { 3700 return ret; 3701 } 3702 } 3703 3704 return 0; 3705 } 3706 3707 void virtio_cleanup(VirtIODevice *vdev) 3708 { 3709 qemu_del_vm_change_state_handler(vdev->vmstate); 3710 } 3711 3712 static void virtio_vmstate_change(void *opaque, bool running, RunState state) 3713 { 3714 VirtIODevice *vdev = opaque; 3715 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3716 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3717 bool backend_run = running && virtio_device_started(vdev, vdev->status); 3718 vdev->vm_running = running; 3719 3720 if (backend_run) { 3721 virtio_set_status(vdev, vdev->status); 3722 } 3723 3724 if (k->vmstate_change) { 3725 k->vmstate_change(qbus->parent, backend_run); 3726 } 3727 3728 if (!backend_run) { 3729 virtio_set_status(vdev, vdev->status); 3730 } 3731 } 3732 3733 void virtio_instance_init_common(Object *proxy_obj, void *data, 3734 size_t vdev_size, const char *vdev_name) 3735 { 3736 DeviceState *vdev = data; 3737 3738 object_initialize_child_with_props(proxy_obj, "virtio-backend", vdev, 3739 vdev_size, vdev_name, &error_abort, 3740 NULL); 3741 qdev_alias_all_properties(vdev, proxy_obj); 3742 } 3743 3744 void virtio_init(VirtIODevice *vdev, uint16_t device_id, size_t config_size) 3745 { 3746 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3747 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3748 int i; 3749 int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0; 3750 3751 if (nvectors) { 3752 vdev->vector_queues = 3753 g_malloc0(sizeof(*vdev->vector_queues) * nvectors); 3754 } 3755 3756 vdev->start_on_kick = false; 3757 vdev->started = false; 3758 vdev->vhost_started = false; 3759 vdev->device_id = device_id; 3760 vdev->status = 0; 3761 qatomic_set(&vdev->isr, 0); 3762 vdev->queue_sel = 0; 3763 vdev->config_vector = VIRTIO_NO_VECTOR; 3764 vdev->vq = g_new0(VirtQueue, VIRTIO_QUEUE_MAX); 3765 vdev->vm_running = runstate_is_running(); 3766 vdev->broken = false; 3767 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 3768 vdev->vq[i].vector = VIRTIO_NO_VECTOR; 3769 vdev->vq[i].vdev = vdev; 3770 vdev->vq[i].queue_index = i; 3771 vdev->vq[i].host_notifier_enabled = false; 3772 } 3773 3774 vdev->name = virtio_id_to_name(device_id); 3775 vdev->config_len = config_size; 3776 if (vdev->config_len) { 3777 vdev->config = g_malloc0(config_size); 3778 } else { 3779 vdev->config = NULL; 3780 } 3781 vdev->vmstate = qdev_add_vm_change_state_handler(DEVICE(vdev), 3782 virtio_vmstate_change, vdev); 3783 vdev->device_endian = virtio_default_endian(); 3784 vdev->use_guest_notifier_mask = true; 3785 } 3786 3787 /* 3788 * Only devices that have already been around prior to defining the virtio 3789 * standard support legacy mode; this includes devices not specified in the 3790 * standard. All newer devices conform to the virtio standard only. 3791 */ 3792 bool virtio_legacy_allowed(VirtIODevice *vdev) 3793 { 3794 switch (vdev->device_id) { 3795 case VIRTIO_ID_NET: 3796 case VIRTIO_ID_BLOCK: 3797 case VIRTIO_ID_CONSOLE: 3798 case VIRTIO_ID_RNG: 3799 case VIRTIO_ID_BALLOON: 3800 case VIRTIO_ID_RPMSG: 3801 case VIRTIO_ID_SCSI: 3802 case VIRTIO_ID_9P: 3803 case VIRTIO_ID_RPROC_SERIAL: 3804 case VIRTIO_ID_CAIF: 3805 return true; 3806 default: 3807 return false; 3808 } 3809 } 3810 3811 bool virtio_legacy_check_disabled(VirtIODevice *vdev) 3812 { 3813 return vdev->disable_legacy_check; 3814 } 3815 3816 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n) 3817 { 3818 return vdev->vq[n].vring.desc; 3819 } 3820 3821 bool virtio_queue_enabled_legacy(VirtIODevice *vdev, int n) 3822 { 3823 return virtio_queue_get_desc_addr(vdev, n) != 0; 3824 } 3825 3826 bool virtio_queue_enabled(VirtIODevice *vdev, int n) 3827 { 3828 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3829 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3830 3831 if (k->queue_enabled) { 3832 return k->queue_enabled(qbus->parent, n); 3833 } 3834 return virtio_queue_enabled_legacy(vdev, n); 3835 } 3836 3837 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n) 3838 { 3839 return vdev->vq[n].vring.avail; 3840 } 3841 3842 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n) 3843 { 3844 return vdev->vq[n].vring.used; 3845 } 3846 3847 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n) 3848 { 3849 return sizeof(VRingDesc) * vdev->vq[n].vring.num; 3850 } 3851 3852 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n) 3853 { 3854 int s; 3855 3856 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3857 return sizeof(struct VRingPackedDescEvent); 3858 } 3859 3860 s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 3861 return offsetof(VRingAvail, ring) + 3862 sizeof(uint16_t) * vdev->vq[n].vring.num + s; 3863 } 3864 3865 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n) 3866 { 3867 int s; 3868 3869 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3870 return sizeof(struct VRingPackedDescEvent); 3871 } 3872 3873 s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 3874 return offsetof(VRingUsed, ring) + 3875 sizeof(VRingUsedElem) * vdev->vq[n].vring.num + s; 3876 } 3877 3878 static unsigned int virtio_queue_packed_get_last_avail_idx(VirtIODevice *vdev, 3879 int n) 3880 { 3881 unsigned int avail, used; 3882 3883 avail = vdev->vq[n].last_avail_idx; 3884 avail |= ((uint16_t)vdev->vq[n].last_avail_wrap_counter) << 15; 3885 3886 used = vdev->vq[n].used_idx; 3887 used |= ((uint16_t)vdev->vq[n].used_wrap_counter) << 15; 3888 3889 return avail | used << 16; 3890 } 3891 3892 static uint16_t virtio_queue_split_get_last_avail_idx(VirtIODevice *vdev, 3893 int n) 3894 { 3895 return vdev->vq[n].last_avail_idx; 3896 } 3897 3898 unsigned int virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n) 3899 { 3900 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3901 return virtio_queue_packed_get_last_avail_idx(vdev, n); 3902 } else { 3903 return virtio_queue_split_get_last_avail_idx(vdev, n); 3904 } 3905 } 3906 3907 static void virtio_queue_packed_set_last_avail_idx(VirtIODevice *vdev, 3908 int n, unsigned int idx) 3909 { 3910 struct VirtQueue *vq = &vdev->vq[n]; 3911 3912 vq->last_avail_idx = vq->shadow_avail_idx = idx & 0x7fff; 3913 vq->last_avail_wrap_counter = 3914 vq->shadow_avail_wrap_counter = !!(idx & 0x8000); 3915 idx >>= 16; 3916 vq->used_idx = idx & 0x7ffff; 3917 vq->used_wrap_counter = !!(idx & 0x8000); 3918 } 3919 3920 static void virtio_queue_split_set_last_avail_idx(VirtIODevice *vdev, 3921 int n, unsigned int idx) 3922 { 3923 vdev->vq[n].last_avail_idx = idx; 3924 vdev->vq[n].shadow_avail_idx = idx; 3925 } 3926 3927 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, 3928 unsigned int idx) 3929 { 3930 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3931 virtio_queue_packed_set_last_avail_idx(vdev, n, idx); 3932 } else { 3933 virtio_queue_split_set_last_avail_idx(vdev, n, idx); 3934 } 3935 } 3936 3937 static void virtio_queue_packed_restore_last_avail_idx(VirtIODevice *vdev, 3938 int n) 3939 { 3940 /* We don't have a reference like avail idx in shared memory */ 3941 return; 3942 } 3943 3944 static void virtio_queue_split_restore_last_avail_idx(VirtIODevice *vdev, 3945 int n) 3946 { 3947 RCU_READ_LOCK_GUARD(); 3948 if (vdev->vq[n].vring.desc) { 3949 vdev->vq[n].last_avail_idx = vring_used_idx(&vdev->vq[n]); 3950 vdev->vq[n].shadow_avail_idx = vdev->vq[n].last_avail_idx; 3951 } 3952 } 3953 3954 void virtio_queue_restore_last_avail_idx(VirtIODevice *vdev, int n) 3955 { 3956 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3957 virtio_queue_packed_restore_last_avail_idx(vdev, n); 3958 } else { 3959 virtio_queue_split_restore_last_avail_idx(vdev, n); 3960 } 3961 } 3962 3963 static void virtio_queue_packed_update_used_idx(VirtIODevice *vdev, int n) 3964 { 3965 /* used idx was updated through set_last_avail_idx() */ 3966 return; 3967 } 3968 3969 static void virtio_split_packed_update_used_idx(VirtIODevice *vdev, int n) 3970 { 3971 RCU_READ_LOCK_GUARD(); 3972 if (vdev->vq[n].vring.desc) { 3973 vdev->vq[n].used_idx = vring_used_idx(&vdev->vq[n]); 3974 } 3975 } 3976 3977 void virtio_queue_update_used_idx(VirtIODevice *vdev, int n) 3978 { 3979 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3980 return virtio_queue_packed_update_used_idx(vdev, n); 3981 } else { 3982 return virtio_split_packed_update_used_idx(vdev, n); 3983 } 3984 } 3985 3986 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n) 3987 { 3988 vdev->vq[n].signalled_used_valid = false; 3989 } 3990 3991 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n) 3992 { 3993 return vdev->vq + n; 3994 } 3995 3996 uint16_t virtio_get_queue_index(VirtQueue *vq) 3997 { 3998 return vq->queue_index; 3999 } 4000 4001 static void virtio_queue_guest_notifier_read(EventNotifier *n) 4002 { 4003 VirtQueue *vq = container_of(n, VirtQueue, guest_notifier); 4004 if (event_notifier_test_and_clear(n)) { 4005 virtio_irq(vq); 4006 } 4007 } 4008 4009 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign, 4010 bool with_irqfd) 4011 { 4012 if (assign && !with_irqfd) { 4013 event_notifier_set_handler(&vq->guest_notifier, 4014 virtio_queue_guest_notifier_read); 4015 } else { 4016 event_notifier_set_handler(&vq->guest_notifier, NULL); 4017 } 4018 if (!assign) { 4019 /* Test and clear notifier before closing it, 4020 * in case poll callback didn't have time to run. */ 4021 virtio_queue_guest_notifier_read(&vq->guest_notifier); 4022 } 4023 } 4024 4025 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq) 4026 { 4027 return &vq->guest_notifier; 4028 } 4029 4030 static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier *n) 4031 { 4032 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 4033 4034 virtio_queue_set_notification(vq, 0); 4035 } 4036 4037 static bool virtio_queue_host_notifier_aio_poll(void *opaque) 4038 { 4039 EventNotifier *n = opaque; 4040 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 4041 4042 return vq->vring.desc && !virtio_queue_empty(vq); 4043 } 4044 4045 static void virtio_queue_host_notifier_aio_poll_ready(EventNotifier *n) 4046 { 4047 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 4048 4049 virtio_queue_notify_vq(vq); 4050 } 4051 4052 static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n) 4053 { 4054 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 4055 4056 /* Caller polls once more after this to catch requests that race with us */ 4057 virtio_queue_set_notification(vq, 1); 4058 } 4059 4060 void virtio_queue_aio_attach_host_notifier(VirtQueue *vq, AioContext *ctx) 4061 { 4062 aio_set_event_notifier(ctx, &vq->host_notifier, true, 4063 virtio_queue_host_notifier_read, 4064 virtio_queue_host_notifier_aio_poll, 4065 virtio_queue_host_notifier_aio_poll_ready); 4066 aio_set_event_notifier_poll(ctx, &vq->host_notifier, 4067 virtio_queue_host_notifier_aio_poll_begin, 4068 virtio_queue_host_notifier_aio_poll_end); 4069 } 4070 4071 /* 4072 * Same as virtio_queue_aio_attach_host_notifier() but without polling. Use 4073 * this for rx virtqueues and similar cases where the virtqueue handler 4074 * function does not pop all elements. When the virtqueue is left non-empty 4075 * polling consumes CPU cycles and should not be used. 4076 */ 4077 void virtio_queue_aio_attach_host_notifier_no_poll(VirtQueue *vq, AioContext *ctx) 4078 { 4079 aio_set_event_notifier(ctx, &vq->host_notifier, true, 4080 virtio_queue_host_notifier_read, 4081 NULL, NULL); 4082 } 4083 4084 void virtio_queue_aio_detach_host_notifier(VirtQueue *vq, AioContext *ctx) 4085 { 4086 aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL, NULL, NULL); 4087 /* Test and clear notifier before after disabling event, 4088 * in case poll callback didn't have time to run. */ 4089 virtio_queue_host_notifier_read(&vq->host_notifier); 4090 } 4091 4092 void virtio_queue_host_notifier_read(EventNotifier *n) 4093 { 4094 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 4095 if (event_notifier_test_and_clear(n)) { 4096 virtio_queue_notify_vq(vq); 4097 } 4098 } 4099 4100 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq) 4101 { 4102 return &vq->host_notifier; 4103 } 4104 4105 void virtio_queue_set_host_notifier_enabled(VirtQueue *vq, bool enabled) 4106 { 4107 vq->host_notifier_enabled = enabled; 4108 } 4109 4110 int virtio_queue_set_host_notifier_mr(VirtIODevice *vdev, int n, 4111 MemoryRegion *mr, bool assign) 4112 { 4113 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 4114 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 4115 4116 if (k->set_host_notifier_mr) { 4117 return k->set_host_notifier_mr(qbus->parent, n, mr, assign); 4118 } 4119 4120 return -1; 4121 } 4122 4123 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name) 4124 { 4125 g_free(vdev->bus_name); 4126 vdev->bus_name = g_strdup(bus_name); 4127 } 4128 4129 void G_GNUC_PRINTF(2, 3) virtio_error(VirtIODevice *vdev, const char *fmt, ...) 4130 { 4131 va_list ap; 4132 4133 va_start(ap, fmt); 4134 error_vreport(fmt, ap); 4135 va_end(ap); 4136 4137 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 4138 vdev->status = vdev->status | VIRTIO_CONFIG_S_NEEDS_RESET; 4139 virtio_notify_config(vdev); 4140 } 4141 4142 vdev->broken = true; 4143 } 4144 4145 static void virtio_memory_listener_commit(MemoryListener *listener) 4146 { 4147 VirtIODevice *vdev = container_of(listener, VirtIODevice, listener); 4148 int i; 4149 4150 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 4151 if (vdev->vq[i].vring.num == 0) { 4152 break; 4153 } 4154 virtio_init_region_cache(vdev, i); 4155 } 4156 } 4157 4158 static void virtio_device_realize(DeviceState *dev, Error **errp) 4159 { 4160 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 4161 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 4162 Error *err = NULL; 4163 4164 /* Devices should either use vmsd or the load/save methods */ 4165 assert(!vdc->vmsd || !vdc->load); 4166 4167 if (vdc->realize != NULL) { 4168 vdc->realize(dev, &err); 4169 if (err != NULL) { 4170 error_propagate(errp, err); 4171 return; 4172 } 4173 } 4174 4175 virtio_bus_device_plugged(vdev, &err); 4176 if (err != NULL) { 4177 error_propagate(errp, err); 4178 vdc->unrealize(dev); 4179 return; 4180 } 4181 4182 vdev->listener.commit = virtio_memory_listener_commit; 4183 vdev->listener.name = "virtio"; 4184 memory_listener_register(&vdev->listener, vdev->dma_as); 4185 QTAILQ_INSERT_TAIL(&virtio_list, vdev, next); 4186 } 4187 4188 static void virtio_device_unrealize(DeviceState *dev) 4189 { 4190 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 4191 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 4192 4193 memory_listener_unregister(&vdev->listener); 4194 virtio_bus_device_unplugged(vdev); 4195 4196 if (vdc->unrealize != NULL) { 4197 vdc->unrealize(dev); 4198 } 4199 4200 QTAILQ_REMOVE(&virtio_list, vdev, next); 4201 g_free(vdev->bus_name); 4202 vdev->bus_name = NULL; 4203 } 4204 4205 static void virtio_device_free_virtqueues(VirtIODevice *vdev) 4206 { 4207 int i; 4208 if (!vdev->vq) { 4209 return; 4210 } 4211 4212 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 4213 if (vdev->vq[i].vring.num == 0) { 4214 break; 4215 } 4216 virtio_virtqueue_reset_region_cache(&vdev->vq[i]); 4217 } 4218 g_free(vdev->vq); 4219 } 4220 4221 static void virtio_device_instance_finalize(Object *obj) 4222 { 4223 VirtIODevice *vdev = VIRTIO_DEVICE(obj); 4224 4225 virtio_device_free_virtqueues(vdev); 4226 4227 g_free(vdev->config); 4228 g_free(vdev->vector_queues); 4229 } 4230 4231 static Property virtio_properties[] = { 4232 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features), 4233 DEFINE_PROP_BOOL("use-started", VirtIODevice, use_started, true), 4234 DEFINE_PROP_BOOL("use-disabled-flag", VirtIODevice, use_disabled_flag, true), 4235 DEFINE_PROP_BOOL("x-disable-legacy-check", VirtIODevice, 4236 disable_legacy_check, false), 4237 DEFINE_PROP_END_OF_LIST(), 4238 }; 4239 4240 static int virtio_device_start_ioeventfd_impl(VirtIODevice *vdev) 4241 { 4242 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev))); 4243 int i, n, r, err; 4244 4245 /* 4246 * Batch all the host notifiers in a single transaction to avoid 4247 * quadratic time complexity in address_space_update_ioeventfds(). 4248 */ 4249 memory_region_transaction_begin(); 4250 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 4251 VirtQueue *vq = &vdev->vq[n]; 4252 if (!virtio_queue_get_num(vdev, n)) { 4253 continue; 4254 } 4255 r = virtio_bus_set_host_notifier(qbus, n, true); 4256 if (r < 0) { 4257 err = r; 4258 goto assign_error; 4259 } 4260 event_notifier_set_handler(&vq->host_notifier, 4261 virtio_queue_host_notifier_read); 4262 } 4263 4264 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 4265 /* Kick right away to begin processing requests already in vring */ 4266 VirtQueue *vq = &vdev->vq[n]; 4267 if (!vq->vring.num) { 4268 continue; 4269 } 4270 event_notifier_set(&vq->host_notifier); 4271 } 4272 memory_region_transaction_commit(); 4273 return 0; 4274 4275 assign_error: 4276 i = n; /* save n for a second iteration after transaction is committed. */ 4277 while (--n >= 0) { 4278 VirtQueue *vq = &vdev->vq[n]; 4279 if (!virtio_queue_get_num(vdev, n)) { 4280 continue; 4281 } 4282 4283 event_notifier_set_handler(&vq->host_notifier, NULL); 4284 r = virtio_bus_set_host_notifier(qbus, n, false); 4285 assert(r >= 0); 4286 } 4287 /* 4288 * The transaction expects the ioeventfds to be open when it 4289 * commits. Do it now, before the cleanup loop. 4290 */ 4291 memory_region_transaction_commit(); 4292 4293 while (--i >= 0) { 4294 if (!virtio_queue_get_num(vdev, i)) { 4295 continue; 4296 } 4297 virtio_bus_cleanup_host_notifier(qbus, i); 4298 } 4299 return err; 4300 } 4301 4302 int virtio_device_start_ioeventfd(VirtIODevice *vdev) 4303 { 4304 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 4305 VirtioBusState *vbus = VIRTIO_BUS(qbus); 4306 4307 return virtio_bus_start_ioeventfd(vbus); 4308 } 4309 4310 static void virtio_device_stop_ioeventfd_impl(VirtIODevice *vdev) 4311 { 4312 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev))); 4313 int n, r; 4314 4315 /* 4316 * Batch all the host notifiers in a single transaction to avoid 4317 * quadratic time complexity in address_space_update_ioeventfds(). 4318 */ 4319 memory_region_transaction_begin(); 4320 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 4321 VirtQueue *vq = &vdev->vq[n]; 4322 4323 if (!virtio_queue_get_num(vdev, n)) { 4324 continue; 4325 } 4326 event_notifier_set_handler(&vq->host_notifier, NULL); 4327 r = virtio_bus_set_host_notifier(qbus, n, false); 4328 assert(r >= 0); 4329 } 4330 /* 4331 * The transaction expects the ioeventfds to be open when it 4332 * commits. Do it now, before the cleanup loop. 4333 */ 4334 memory_region_transaction_commit(); 4335 4336 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 4337 if (!virtio_queue_get_num(vdev, n)) { 4338 continue; 4339 } 4340 virtio_bus_cleanup_host_notifier(qbus, n); 4341 } 4342 } 4343 4344 int virtio_device_grab_ioeventfd(VirtIODevice *vdev) 4345 { 4346 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 4347 VirtioBusState *vbus = VIRTIO_BUS(qbus); 4348 4349 return virtio_bus_grab_ioeventfd(vbus); 4350 } 4351 4352 void virtio_device_release_ioeventfd(VirtIODevice *vdev) 4353 { 4354 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 4355 VirtioBusState *vbus = VIRTIO_BUS(qbus); 4356 4357 virtio_bus_release_ioeventfd(vbus); 4358 } 4359 4360 static void virtio_device_class_init(ObjectClass *klass, void *data) 4361 { 4362 /* Set the default value here. */ 4363 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 4364 DeviceClass *dc = DEVICE_CLASS(klass); 4365 4366 dc->realize = virtio_device_realize; 4367 dc->unrealize = virtio_device_unrealize; 4368 dc->bus_type = TYPE_VIRTIO_BUS; 4369 device_class_set_props(dc, virtio_properties); 4370 vdc->start_ioeventfd = virtio_device_start_ioeventfd_impl; 4371 vdc->stop_ioeventfd = virtio_device_stop_ioeventfd_impl; 4372 4373 vdc->legacy_features |= VIRTIO_LEGACY_FEATURES; 4374 4375 QTAILQ_INIT(&virtio_list); 4376 } 4377 4378 bool virtio_device_ioeventfd_enabled(VirtIODevice *vdev) 4379 { 4380 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 4381 VirtioBusState *vbus = VIRTIO_BUS(qbus); 4382 4383 return virtio_bus_ioeventfd_enabled(vbus); 4384 } 4385 4386 VirtioInfoList *qmp_x_query_virtio(Error **errp) 4387 { 4388 VirtioInfoList *list = NULL; 4389 VirtioInfoList *node; 4390 VirtIODevice *vdev; 4391 4392 QTAILQ_FOREACH(vdev, &virtio_list, next) { 4393 DeviceState *dev = DEVICE(vdev); 4394 Error *err = NULL; 4395 QObject *obj = qmp_qom_get(dev->canonical_path, "realized", &err); 4396 4397 if (err == NULL) { 4398 GString *is_realized = qobject_to_json_pretty(obj, true); 4399 /* virtio device is NOT realized, remove it from list */ 4400 if (!strncmp(is_realized->str, "false", 4)) { 4401 QTAILQ_REMOVE(&virtio_list, vdev, next); 4402 } else { 4403 node = g_new0(VirtioInfoList, 1); 4404 node->value = g_new(VirtioInfo, 1); 4405 node->value->path = g_strdup(dev->canonical_path); 4406 node->value->name = g_strdup(vdev->name); 4407 QAPI_LIST_PREPEND(list, node->value); 4408 } 4409 g_string_free(is_realized, true); 4410 } 4411 qobject_unref(obj); 4412 } 4413 4414 return list; 4415 } 4416 4417 static VirtIODevice *virtio_device_find(const char *path) 4418 { 4419 VirtIODevice *vdev; 4420 4421 QTAILQ_FOREACH(vdev, &virtio_list, next) { 4422 DeviceState *dev = DEVICE(vdev); 4423 4424 if (strcmp(dev->canonical_path, path) != 0) { 4425 continue; 4426 } 4427 4428 Error *err = NULL; 4429 QObject *obj = qmp_qom_get(dev->canonical_path, "realized", &err); 4430 if (err == NULL) { 4431 GString *is_realized = qobject_to_json_pretty(obj, true); 4432 /* virtio device is NOT realized, remove it from list */ 4433 if (!strncmp(is_realized->str, "false", 4)) { 4434 g_string_free(is_realized, true); 4435 qobject_unref(obj); 4436 QTAILQ_REMOVE(&virtio_list, vdev, next); 4437 return NULL; 4438 } 4439 g_string_free(is_realized, true); 4440 } else { 4441 /* virtio device doesn't exist in QOM tree */ 4442 QTAILQ_REMOVE(&virtio_list, vdev, next); 4443 qobject_unref(obj); 4444 return NULL; 4445 } 4446 /* device exists in QOM tree & is realized */ 4447 qobject_unref(obj); 4448 return vdev; 4449 } 4450 return NULL; 4451 } 4452 4453 #define CONVERT_FEATURES(type, map, is_status, bitmap) \ 4454 ({ \ 4455 type *list = NULL; \ 4456 type *node; \ 4457 for (i = 0; map[i].virtio_bit != -1; i++) { \ 4458 if (is_status) { \ 4459 bit = map[i].virtio_bit; \ 4460 } \ 4461 else { \ 4462 bit = 1ULL << map[i].virtio_bit; \ 4463 } \ 4464 if ((bitmap & bit) == 0) { \ 4465 continue; \ 4466 } \ 4467 node = g_new0(type, 1); \ 4468 node->value = g_strdup(map[i].feature_desc); \ 4469 node->next = list; \ 4470 list = node; \ 4471 bitmap ^= bit; \ 4472 } \ 4473 list; \ 4474 }) 4475 4476 static VirtioDeviceStatus *qmp_decode_status(uint8_t bitmap) 4477 { 4478 VirtioDeviceStatus *status; 4479 uint8_t bit; 4480 int i; 4481 4482 status = g_new0(VirtioDeviceStatus, 1); 4483 status->statuses = CONVERT_FEATURES(strList, virtio_config_status_map, 4484 1, bitmap); 4485 status->has_unknown_statuses = bitmap != 0; 4486 if (status->has_unknown_statuses) { 4487 status->unknown_statuses = bitmap; 4488 } 4489 4490 return status; 4491 } 4492 4493 static VhostDeviceProtocols *qmp_decode_protocols(uint64_t bitmap) 4494 { 4495 VhostDeviceProtocols *vhu_protocols; 4496 uint64_t bit; 4497 int i; 4498 4499 vhu_protocols = g_new0(VhostDeviceProtocols, 1); 4500 vhu_protocols->protocols = 4501 CONVERT_FEATURES(strList, 4502 vhost_user_protocol_map, 0, bitmap); 4503 vhu_protocols->has_unknown_protocols = bitmap != 0; 4504 if (vhu_protocols->has_unknown_protocols) { 4505 vhu_protocols->unknown_protocols = bitmap; 4506 } 4507 4508 return vhu_protocols; 4509 } 4510 4511 static VirtioDeviceFeatures *qmp_decode_features(uint16_t device_id, 4512 uint64_t bitmap) 4513 { 4514 VirtioDeviceFeatures *features; 4515 uint64_t bit; 4516 int i; 4517 4518 features = g_new0(VirtioDeviceFeatures, 1); 4519 features->has_dev_features = true; 4520 4521 /* transport features */ 4522 features->transports = CONVERT_FEATURES(strList, virtio_transport_map, 0, 4523 bitmap); 4524 4525 /* device features */ 4526 switch (device_id) { 4527 #ifdef CONFIG_VIRTIO_SERIAL 4528 case VIRTIO_ID_CONSOLE: 4529 features->dev_features = 4530 CONVERT_FEATURES(strList, virtio_serial_feature_map, 0, bitmap); 4531 break; 4532 #endif 4533 #ifdef CONFIG_VIRTIO_BLK 4534 case VIRTIO_ID_BLOCK: 4535 features->dev_features = 4536 CONVERT_FEATURES(strList, virtio_blk_feature_map, 0, bitmap); 4537 break; 4538 #endif 4539 #ifdef CONFIG_VIRTIO_GPU 4540 case VIRTIO_ID_GPU: 4541 features->dev_features = 4542 CONVERT_FEATURES(strList, virtio_gpu_feature_map, 0, bitmap); 4543 break; 4544 #endif 4545 #ifdef CONFIG_VIRTIO_NET 4546 case VIRTIO_ID_NET: 4547 features->dev_features = 4548 CONVERT_FEATURES(strList, virtio_net_feature_map, 0, bitmap); 4549 break; 4550 #endif 4551 #ifdef CONFIG_VIRTIO_SCSI 4552 case VIRTIO_ID_SCSI: 4553 features->dev_features = 4554 CONVERT_FEATURES(strList, virtio_scsi_feature_map, 0, bitmap); 4555 break; 4556 #endif 4557 #ifdef CONFIG_VIRTIO_BALLOON 4558 case VIRTIO_ID_BALLOON: 4559 features->dev_features = 4560 CONVERT_FEATURES(strList, virtio_balloon_feature_map, 0, bitmap); 4561 break; 4562 #endif 4563 #ifdef CONFIG_VIRTIO_IOMMU 4564 case VIRTIO_ID_IOMMU: 4565 features->dev_features = 4566 CONVERT_FEATURES(strList, virtio_iommu_feature_map, 0, bitmap); 4567 break; 4568 #endif 4569 #ifdef CONFIG_VIRTIO_INPUT 4570 case VIRTIO_ID_INPUT: 4571 features->dev_features = 4572 CONVERT_FEATURES(strList, virtio_input_feature_map, 0, bitmap); 4573 break; 4574 #endif 4575 #ifdef CONFIG_VHOST_USER_FS 4576 case VIRTIO_ID_FS: 4577 features->dev_features = 4578 CONVERT_FEATURES(strList, virtio_fs_feature_map, 0, bitmap); 4579 break; 4580 #endif 4581 #ifdef CONFIG_VHOST_VSOCK 4582 case VIRTIO_ID_VSOCK: 4583 features->dev_features = 4584 CONVERT_FEATURES(strList, virtio_vsock_feature_map, 0, bitmap); 4585 break; 4586 #endif 4587 #ifdef CONFIG_VIRTIO_CRYPTO 4588 case VIRTIO_ID_CRYPTO: 4589 features->dev_features = 4590 CONVERT_FEATURES(strList, virtio_crypto_feature_map, 0, bitmap); 4591 break; 4592 #endif 4593 #ifdef CONFIG_VIRTIO_MEM 4594 case VIRTIO_ID_MEM: 4595 features->dev_features = 4596 CONVERT_FEATURES(strList, virtio_mem_feature_map, 0, bitmap); 4597 break; 4598 #endif 4599 #ifdef CONFIG_VIRTIO_I2C_ADAPTER 4600 case VIRTIO_ID_I2C_ADAPTER: 4601 features->dev_features = 4602 CONVERT_FEATURES(strList, virtio_i2c_feature_map, 0, bitmap); 4603 break; 4604 #endif 4605 #ifdef CONFIG_VIRTIO_RNG 4606 case VIRTIO_ID_RNG: 4607 features->dev_features = 4608 CONVERT_FEATURES(strList, virtio_rng_feature_map, 0, bitmap); 4609 break; 4610 #endif 4611 /* No features */ 4612 case VIRTIO_ID_9P: 4613 case VIRTIO_ID_PMEM: 4614 case VIRTIO_ID_IOMEM: 4615 case VIRTIO_ID_RPMSG: 4616 case VIRTIO_ID_CLOCK: 4617 case VIRTIO_ID_MAC80211_WLAN: 4618 case VIRTIO_ID_MAC80211_HWSIM: 4619 case VIRTIO_ID_RPROC_SERIAL: 4620 case VIRTIO_ID_MEMORY_BALLOON: 4621 case VIRTIO_ID_CAIF: 4622 case VIRTIO_ID_SIGNAL_DIST: 4623 case VIRTIO_ID_PSTORE: 4624 case VIRTIO_ID_SOUND: 4625 case VIRTIO_ID_BT: 4626 case VIRTIO_ID_RPMB: 4627 case VIRTIO_ID_VIDEO_ENCODER: 4628 case VIRTIO_ID_VIDEO_DECODER: 4629 case VIRTIO_ID_SCMI: 4630 case VIRTIO_ID_NITRO_SEC_MOD: 4631 case VIRTIO_ID_WATCHDOG: 4632 case VIRTIO_ID_CAN: 4633 case VIRTIO_ID_DMABUF: 4634 case VIRTIO_ID_PARAM_SERV: 4635 case VIRTIO_ID_AUDIO_POLICY: 4636 case VIRTIO_ID_GPIO: 4637 break; 4638 default: 4639 g_assert_not_reached(); 4640 } 4641 4642 features->has_unknown_dev_features = bitmap != 0; 4643 if (features->has_unknown_dev_features) { 4644 features->unknown_dev_features = bitmap; 4645 } 4646 4647 return features; 4648 } 4649 4650 VirtioStatus *qmp_x_query_virtio_status(const char *path, Error **errp) 4651 { 4652 VirtIODevice *vdev; 4653 VirtioStatus *status; 4654 4655 vdev = virtio_device_find(path); 4656 if (vdev == NULL) { 4657 error_setg(errp, "Path %s is not a VirtIODevice", path); 4658 return NULL; 4659 } 4660 4661 status = g_new0(VirtioStatus, 1); 4662 status->name = g_strdup(vdev->name); 4663 status->device_id = vdev->device_id; 4664 status->vhost_started = vdev->vhost_started; 4665 status->guest_features = qmp_decode_features(vdev->device_id, 4666 vdev->guest_features); 4667 status->host_features = qmp_decode_features(vdev->device_id, 4668 vdev->host_features); 4669 status->backend_features = qmp_decode_features(vdev->device_id, 4670 vdev->backend_features); 4671 4672 switch (vdev->device_endian) { 4673 case VIRTIO_DEVICE_ENDIAN_LITTLE: 4674 status->device_endian = g_strdup("little"); 4675 break; 4676 case VIRTIO_DEVICE_ENDIAN_BIG: 4677 status->device_endian = g_strdup("big"); 4678 break; 4679 default: 4680 status->device_endian = g_strdup("unknown"); 4681 break; 4682 } 4683 4684 status->num_vqs = virtio_get_num_queues(vdev); 4685 status->status = qmp_decode_status(vdev->status); 4686 status->isr = vdev->isr; 4687 status->queue_sel = vdev->queue_sel; 4688 status->vm_running = vdev->vm_running; 4689 status->broken = vdev->broken; 4690 status->disabled = vdev->disabled; 4691 status->use_started = vdev->use_started; 4692 status->started = vdev->started; 4693 status->start_on_kick = vdev->start_on_kick; 4694 status->disable_legacy_check = vdev->disable_legacy_check; 4695 status->bus_name = g_strdup(vdev->bus_name); 4696 status->use_guest_notifier_mask = vdev->use_guest_notifier_mask; 4697 status->has_vhost_dev = vdev->vhost_started; 4698 4699 if (vdev->vhost_started) { 4700 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 4701 struct vhost_dev *hdev = vdc->get_vhost(vdev); 4702 4703 status->vhost_dev = g_new0(VhostStatus, 1); 4704 status->vhost_dev->n_mem_sections = hdev->n_mem_sections; 4705 status->vhost_dev->n_tmp_sections = hdev->n_tmp_sections; 4706 status->vhost_dev->nvqs = hdev->nvqs; 4707 status->vhost_dev->vq_index = hdev->vq_index; 4708 status->vhost_dev->features = 4709 qmp_decode_features(vdev->device_id, hdev->features); 4710 status->vhost_dev->acked_features = 4711 qmp_decode_features(vdev->device_id, hdev->acked_features); 4712 status->vhost_dev->backend_features = 4713 qmp_decode_features(vdev->device_id, hdev->backend_features); 4714 status->vhost_dev->protocol_features = 4715 qmp_decode_protocols(hdev->protocol_features); 4716 status->vhost_dev->max_queues = hdev->max_queues; 4717 status->vhost_dev->backend_cap = hdev->backend_cap; 4718 status->vhost_dev->log_enabled = hdev->log_enabled; 4719 status->vhost_dev->log_size = hdev->log_size; 4720 } 4721 4722 return status; 4723 } 4724 4725 VirtVhostQueueStatus *qmp_x_query_virtio_vhost_queue_status(const char *path, 4726 uint16_t queue, 4727 Error **errp) 4728 { 4729 VirtIODevice *vdev; 4730 VirtVhostQueueStatus *status; 4731 4732 vdev = virtio_device_find(path); 4733 if (vdev == NULL) { 4734 error_setg(errp, "Path %s is not a VirtIODevice", path); 4735 return NULL; 4736 } 4737 4738 if (!vdev->vhost_started) { 4739 error_setg(errp, "Error: vhost device has not started yet"); 4740 return NULL; 4741 } 4742 4743 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 4744 struct vhost_dev *hdev = vdc->get_vhost(vdev); 4745 4746 if (queue < hdev->vq_index || queue >= hdev->vq_index + hdev->nvqs) { 4747 error_setg(errp, "Invalid vhost virtqueue number %d", queue); 4748 return NULL; 4749 } 4750 4751 status = g_new0(VirtVhostQueueStatus, 1); 4752 status->name = g_strdup(vdev->name); 4753 status->kick = hdev->vqs[queue].kick; 4754 status->call = hdev->vqs[queue].call; 4755 status->desc = (uintptr_t)hdev->vqs[queue].desc; 4756 status->avail = (uintptr_t)hdev->vqs[queue].avail; 4757 status->used = (uintptr_t)hdev->vqs[queue].used; 4758 status->num = hdev->vqs[queue].num; 4759 status->desc_phys = hdev->vqs[queue].desc_phys; 4760 status->desc_size = hdev->vqs[queue].desc_size; 4761 status->avail_phys = hdev->vqs[queue].avail_phys; 4762 status->avail_size = hdev->vqs[queue].avail_size; 4763 status->used_phys = hdev->vqs[queue].used_phys; 4764 status->used_size = hdev->vqs[queue].used_size; 4765 4766 return status; 4767 } 4768 4769 VirtQueueStatus *qmp_x_query_virtio_queue_status(const char *path, 4770 uint16_t queue, 4771 Error **errp) 4772 { 4773 VirtIODevice *vdev; 4774 VirtQueueStatus *status; 4775 4776 vdev = virtio_device_find(path); 4777 if (vdev == NULL) { 4778 error_setg(errp, "Path %s is not a VirtIODevice", path); 4779 return NULL; 4780 } 4781 4782 if (queue >= VIRTIO_QUEUE_MAX || !virtio_queue_get_num(vdev, queue)) { 4783 error_setg(errp, "Invalid virtqueue number %d", queue); 4784 return NULL; 4785 } 4786 4787 status = g_new0(VirtQueueStatus, 1); 4788 status->name = g_strdup(vdev->name); 4789 status->queue_index = vdev->vq[queue].queue_index; 4790 status->inuse = vdev->vq[queue].inuse; 4791 status->vring_num = vdev->vq[queue].vring.num; 4792 status->vring_num_default = vdev->vq[queue].vring.num_default; 4793 status->vring_align = vdev->vq[queue].vring.align; 4794 status->vring_desc = vdev->vq[queue].vring.desc; 4795 status->vring_avail = vdev->vq[queue].vring.avail; 4796 status->vring_used = vdev->vq[queue].vring.used; 4797 status->used_idx = vdev->vq[queue].used_idx; 4798 status->signalled_used = vdev->vq[queue].signalled_used; 4799 status->signalled_used_valid = vdev->vq[queue].signalled_used_valid; 4800 4801 if (vdev->vhost_started) { 4802 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 4803 struct vhost_dev *hdev = vdc->get_vhost(vdev); 4804 4805 /* check if vq index exists for vhost as well */ 4806 if (queue >= hdev->vq_index && queue < hdev->vq_index + hdev->nvqs) { 4807 status->has_last_avail_idx = true; 4808 4809 int vhost_vq_index = 4810 hdev->vhost_ops->vhost_get_vq_index(hdev, queue); 4811 struct vhost_vring_state state = { 4812 .index = vhost_vq_index, 4813 }; 4814 4815 status->last_avail_idx = 4816 hdev->vhost_ops->vhost_get_vring_base(hdev, &state); 4817 } 4818 } else { 4819 status->has_shadow_avail_idx = true; 4820 status->has_last_avail_idx = true; 4821 status->last_avail_idx = vdev->vq[queue].last_avail_idx; 4822 status->shadow_avail_idx = vdev->vq[queue].shadow_avail_idx; 4823 } 4824 4825 return status; 4826 } 4827 4828 static strList *qmp_decode_vring_desc_flags(uint16_t flags) 4829 { 4830 strList *list = NULL; 4831 strList *node; 4832 int i; 4833 4834 struct { 4835 uint16_t flag; 4836 const char *value; 4837 } map[] = { 4838 { VRING_DESC_F_NEXT, "next" }, 4839 { VRING_DESC_F_WRITE, "write" }, 4840 { VRING_DESC_F_INDIRECT, "indirect" }, 4841 { 1 << VRING_PACKED_DESC_F_AVAIL, "avail" }, 4842 { 1 << VRING_PACKED_DESC_F_USED, "used" }, 4843 { 0, "" } 4844 }; 4845 4846 for (i = 0; map[i].flag; i++) { 4847 if ((map[i].flag & flags) == 0) { 4848 continue; 4849 } 4850 node = g_malloc0(sizeof(strList)); 4851 node->value = g_strdup(map[i].value); 4852 node->next = list; 4853 list = node; 4854 } 4855 4856 return list; 4857 } 4858 4859 VirtioQueueElement *qmp_x_query_virtio_queue_element(const char *path, 4860 uint16_t queue, 4861 bool has_index, 4862 uint16_t index, 4863 Error **errp) 4864 { 4865 VirtIODevice *vdev; 4866 VirtQueue *vq; 4867 VirtioQueueElement *element = NULL; 4868 4869 vdev = virtio_device_find(path); 4870 if (vdev == NULL) { 4871 error_setg(errp, "Path %s is not a VirtIO device", path); 4872 return NULL; 4873 } 4874 4875 if (queue >= VIRTIO_QUEUE_MAX || !virtio_queue_get_num(vdev, queue)) { 4876 error_setg(errp, "Invalid virtqueue number %d", queue); 4877 return NULL; 4878 } 4879 vq = &vdev->vq[queue]; 4880 4881 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 4882 error_setg(errp, "Packed ring not supported"); 4883 return NULL; 4884 } else { 4885 unsigned int head, i, max; 4886 VRingMemoryRegionCaches *caches; 4887 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 4888 MemoryRegionCache *desc_cache; 4889 VRingDesc desc; 4890 VirtioRingDescList *list = NULL; 4891 VirtioRingDescList *node; 4892 int rc; int ndescs; 4893 4894 RCU_READ_LOCK_GUARD(); 4895 4896 max = vq->vring.num; 4897 4898 if (!has_index) { 4899 head = vring_avail_ring(vq, vq->last_avail_idx % vq->vring.num); 4900 } else { 4901 head = vring_avail_ring(vq, index % vq->vring.num); 4902 } 4903 i = head; 4904 4905 caches = vring_get_region_caches(vq); 4906 if (!caches) { 4907 error_setg(errp, "Region caches not initialized"); 4908 return NULL; 4909 } 4910 if (caches->desc.len < max * sizeof(VRingDesc)) { 4911 error_setg(errp, "Cannot map descriptor ring"); 4912 return NULL; 4913 } 4914 4915 desc_cache = &caches->desc; 4916 vring_split_desc_read(vdev, &desc, desc_cache, i); 4917 if (desc.flags & VRING_DESC_F_INDIRECT) { 4918 int64_t len; 4919 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, 4920 desc.addr, desc.len, false); 4921 desc_cache = &indirect_desc_cache; 4922 if (len < desc.len) { 4923 error_setg(errp, "Cannot map indirect buffer"); 4924 goto done; 4925 } 4926 4927 max = desc.len / sizeof(VRingDesc); 4928 i = 0; 4929 vring_split_desc_read(vdev, &desc, desc_cache, i); 4930 } 4931 4932 element = g_new0(VirtioQueueElement, 1); 4933 element->avail = g_new0(VirtioRingAvail, 1); 4934 element->used = g_new0(VirtioRingUsed, 1); 4935 element->name = g_strdup(vdev->name); 4936 element->index = head; 4937 element->avail->flags = vring_avail_flags(vq); 4938 element->avail->idx = vring_avail_idx(vq); 4939 element->avail->ring = head; 4940 element->used->flags = vring_used_flags(vq); 4941 element->used->idx = vring_used_idx(vq); 4942 ndescs = 0; 4943 4944 do { 4945 /* A buggy driver may produce an infinite loop */ 4946 if (ndescs >= max) { 4947 break; 4948 } 4949 node = g_new0(VirtioRingDescList, 1); 4950 node->value = g_new0(VirtioRingDesc, 1); 4951 node->value->addr = desc.addr; 4952 node->value->len = desc.len; 4953 node->value->flags = qmp_decode_vring_desc_flags(desc.flags); 4954 node->next = list; 4955 list = node; 4956 4957 ndescs++; 4958 rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, 4959 max, &i); 4960 } while (rc == VIRTQUEUE_READ_DESC_MORE); 4961 element->descs = list; 4962 done: 4963 address_space_cache_destroy(&indirect_desc_cache); 4964 } 4965 4966 return element; 4967 } 4968 4969 static const TypeInfo virtio_device_info = { 4970 .name = TYPE_VIRTIO_DEVICE, 4971 .parent = TYPE_DEVICE, 4972 .instance_size = sizeof(VirtIODevice), 4973 .class_init = virtio_device_class_init, 4974 .instance_finalize = virtio_device_instance_finalize, 4975 .abstract = true, 4976 .class_size = sizeof(VirtioDeviceClass), 4977 }; 4978 4979 static void virtio_register_types(void) 4980 { 4981 type_register_static(&virtio_device_info); 4982 } 4983 4984 type_init(virtio_register_types) 4985