1 /* 2 * Virtio Support 3 * 4 * Copyright IBM, Corp. 2007 5 * 6 * Authors: 7 * Anthony Liguori <aliguori@us.ibm.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 */ 13 14 #include "qemu/osdep.h" 15 #include "qapi/error.h" 16 #include "qapi/qmp/qdict.h" 17 #include "qapi/qapi-commands-virtio.h" 18 #include "qapi/qapi-commands-qom.h" 19 #include "qapi/qapi-visit-virtio.h" 20 #include "qapi/qmp/qjson.h" 21 #include "cpu.h" 22 #include "trace.h" 23 #include "qemu/error-report.h" 24 #include "qemu/log.h" 25 #include "qemu/main-loop.h" 26 #include "qemu/module.h" 27 #include "qom/object_interfaces.h" 28 #include "hw/virtio/virtio.h" 29 #include "migration/qemu-file-types.h" 30 #include "qemu/atomic.h" 31 #include "hw/virtio/virtio-bus.h" 32 #include "hw/qdev-properties.h" 33 #include "hw/virtio/virtio-access.h" 34 #include "sysemu/dma.h" 35 #include "sysemu/runstate.h" 36 #include "standard-headers/linux/virtio_ids.h" 37 #include "standard-headers/linux/vhost_types.h" 38 #include "standard-headers/linux/virtio_blk.h" 39 #include "standard-headers/linux/virtio_console.h" 40 #include "standard-headers/linux/virtio_gpu.h" 41 #include "standard-headers/linux/virtio_net.h" 42 #include "standard-headers/linux/virtio_scsi.h" 43 #include "standard-headers/linux/virtio_i2c.h" 44 #include "standard-headers/linux/virtio_balloon.h" 45 #include "standard-headers/linux/virtio_iommu.h" 46 #include "standard-headers/linux/virtio_mem.h" 47 #include "standard-headers/linux/virtio_vsock.h" 48 #include CONFIG_DEVICES 49 50 /* QAPI list of realized VirtIODevices */ 51 static QTAILQ_HEAD(, VirtIODevice) virtio_list; 52 53 /* 54 * Maximum size of virtio device config space 55 */ 56 #define VHOST_USER_MAX_CONFIG_SIZE 256 57 58 #define FEATURE_ENTRY(name, desc) (qmp_virtio_feature_map_t) \ 59 { .virtio_bit = name, .feature_desc = desc } 60 61 enum VhostUserProtocolFeature { 62 VHOST_USER_PROTOCOL_F_MQ = 0, 63 VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1, 64 VHOST_USER_PROTOCOL_F_RARP = 2, 65 VHOST_USER_PROTOCOL_F_REPLY_ACK = 3, 66 VHOST_USER_PROTOCOL_F_NET_MTU = 4, 67 VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5, 68 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6, 69 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7, 70 VHOST_USER_PROTOCOL_F_PAGEFAULT = 8, 71 VHOST_USER_PROTOCOL_F_CONFIG = 9, 72 VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10, 73 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11, 74 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD = 12, 75 VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13, 76 VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS = 14, 77 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS = 15, 78 VHOST_USER_PROTOCOL_F_MAX 79 }; 80 81 /* Virtio transport features mapping */ 82 static qmp_virtio_feature_map_t virtio_transport_map[] = { 83 /* Virtio device transport features */ 84 #ifndef VIRTIO_CONFIG_NO_LEGACY 85 FEATURE_ENTRY(VIRTIO_F_NOTIFY_ON_EMPTY, \ 86 "VIRTIO_F_NOTIFY_ON_EMPTY: Notify when device runs out of avail. " 87 "descs. on VQ"), 88 FEATURE_ENTRY(VIRTIO_F_ANY_LAYOUT, \ 89 "VIRTIO_F_ANY_LAYOUT: Device accepts arbitrary desc. layouts"), 90 #endif /* !VIRTIO_CONFIG_NO_LEGACY */ 91 FEATURE_ENTRY(VIRTIO_F_VERSION_1, \ 92 "VIRTIO_F_VERSION_1: Device compliant for v1 spec (legacy)"), 93 FEATURE_ENTRY(VIRTIO_F_IOMMU_PLATFORM, \ 94 "VIRTIO_F_IOMMU_PLATFORM: Device can be used on IOMMU platform"), 95 FEATURE_ENTRY(VIRTIO_F_RING_PACKED, \ 96 "VIRTIO_F_RING_PACKED: Device supports packed VQ layout"), 97 FEATURE_ENTRY(VIRTIO_F_IN_ORDER, \ 98 "VIRTIO_F_IN_ORDER: Device uses buffers in same order as made " 99 "available by driver"), 100 FEATURE_ENTRY(VIRTIO_F_ORDER_PLATFORM, \ 101 "VIRTIO_F_ORDER_PLATFORM: Memory accesses ordered by platform"), 102 FEATURE_ENTRY(VIRTIO_F_SR_IOV, \ 103 "VIRTIO_F_SR_IOV: Device supports single root I/O virtualization"), 104 /* Virtio ring transport features */ 105 FEATURE_ENTRY(VIRTIO_RING_F_INDIRECT_DESC, \ 106 "VIRTIO_RING_F_INDIRECT_DESC: Indirect descriptors supported"), 107 FEATURE_ENTRY(VIRTIO_RING_F_EVENT_IDX, \ 108 "VIRTIO_RING_F_EVENT_IDX: Used & avail. event fields enabled"), 109 { -1, "" } 110 }; 111 112 /* Vhost-user protocol features mapping */ 113 static qmp_virtio_feature_map_t vhost_user_protocol_map[] = { 114 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_MQ, \ 115 "VHOST_USER_PROTOCOL_F_MQ: Multiqueue protocol supported"), 116 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_LOG_SHMFD, \ 117 "VHOST_USER_PROTOCOL_F_LOG_SHMFD: Shared log memory fd supported"), 118 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_RARP, \ 119 "VHOST_USER_PROTOCOL_F_RARP: Vhost-user back-end RARP broadcasting " 120 "supported"), 121 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_REPLY_ACK, \ 122 "VHOST_USER_PROTOCOL_F_REPLY_ACK: Requested operation status ack. " 123 "supported"), 124 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_NET_MTU, \ 125 "VHOST_USER_PROTOCOL_F_NET_MTU: Expose host MTU to guest supported"), 126 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_SLAVE_REQ, \ 127 "VHOST_USER_PROTOCOL_F_SLAVE_REQ: Socket fd for back-end initiated " 128 "requests supported"), 129 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_CROSS_ENDIAN, \ 130 "VHOST_USER_PROTOCOL_F_CROSS_ENDIAN: Endianness of VQs for legacy " 131 "devices supported"), 132 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_CRYPTO_SESSION, \ 133 "VHOST_USER_PROTOCOL_F_CRYPTO_SESSION: Session creation for crypto " 134 "operations supported"), 135 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_PAGEFAULT, \ 136 "VHOST_USER_PROTOCOL_F_PAGEFAULT: Request servicing on userfaultfd " 137 "for accessed pages supported"), 138 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_CONFIG, \ 139 "VHOST_USER_PROTOCOL_F_CONFIG: Vhost-user messaging for virtio " 140 "device configuration space supported"), 141 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD, \ 142 "VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD: Slave fd communication " 143 "channel supported"), 144 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_HOST_NOTIFIER, \ 145 "VHOST_USER_PROTOCOL_F_HOST_NOTIFIER: Host notifiers for specified " 146 "VQs supported"), 147 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD, \ 148 "VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD: Shared inflight I/O buffers " 149 "supported"), 150 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_RESET_DEVICE, \ 151 "VHOST_USER_PROTOCOL_F_RESET_DEVICE: Disabling all rings and " 152 "resetting internal device state supported"), 153 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS, \ 154 "VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS: In-band messaging " 155 "supported"), 156 FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS, \ 157 "VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS: Configuration for " 158 "memory slots supported"), 159 { -1, "" } 160 }; 161 162 /* virtio device configuration statuses */ 163 static qmp_virtio_feature_map_t virtio_config_status_map[] = { 164 FEATURE_ENTRY(VIRTIO_CONFIG_S_DRIVER_OK, \ 165 "VIRTIO_CONFIG_S_DRIVER_OK: Driver setup and ready"), 166 FEATURE_ENTRY(VIRTIO_CONFIG_S_FEATURES_OK, \ 167 "VIRTIO_CONFIG_S_FEATURES_OK: Feature negotiation complete"), 168 FEATURE_ENTRY(VIRTIO_CONFIG_S_DRIVER, \ 169 "VIRTIO_CONFIG_S_DRIVER: Guest OS compatible with device"), 170 FEATURE_ENTRY(VIRTIO_CONFIG_S_NEEDS_RESET, \ 171 "VIRTIO_CONFIG_S_NEEDS_RESET: Irrecoverable error, device needs " 172 "reset"), 173 FEATURE_ENTRY(VIRTIO_CONFIG_S_FAILED, \ 174 "VIRTIO_CONFIG_S_FAILED: Error in guest, device failed"), 175 FEATURE_ENTRY(VIRTIO_CONFIG_S_ACKNOWLEDGE, \ 176 "VIRTIO_CONFIG_S_ACKNOWLEDGE: Valid virtio device found"), 177 { -1, "" } 178 }; 179 180 /* virtio-blk features mapping */ 181 qmp_virtio_feature_map_t virtio_blk_feature_map[] = { 182 FEATURE_ENTRY(VIRTIO_BLK_F_SIZE_MAX, \ 183 "VIRTIO_BLK_F_SIZE_MAX: Max segment size is size_max"), 184 FEATURE_ENTRY(VIRTIO_BLK_F_SEG_MAX, \ 185 "VIRTIO_BLK_F_SEG_MAX: Max segments in a request is seg_max"), 186 FEATURE_ENTRY(VIRTIO_BLK_F_GEOMETRY, \ 187 "VIRTIO_BLK_F_GEOMETRY: Legacy geometry available"), 188 FEATURE_ENTRY(VIRTIO_BLK_F_RO, \ 189 "VIRTIO_BLK_F_RO: Device is read-only"), 190 FEATURE_ENTRY(VIRTIO_BLK_F_BLK_SIZE, \ 191 "VIRTIO_BLK_F_BLK_SIZE: Block size of disk available"), 192 FEATURE_ENTRY(VIRTIO_BLK_F_TOPOLOGY, \ 193 "VIRTIO_BLK_F_TOPOLOGY: Topology information available"), 194 FEATURE_ENTRY(VIRTIO_BLK_F_MQ, \ 195 "VIRTIO_BLK_F_MQ: Multiqueue supported"), 196 FEATURE_ENTRY(VIRTIO_BLK_F_DISCARD, \ 197 "VIRTIO_BLK_F_DISCARD: Discard command supported"), 198 FEATURE_ENTRY(VIRTIO_BLK_F_WRITE_ZEROES, \ 199 "VIRTIO_BLK_F_WRITE_ZEROES: Write zeroes command supported"), 200 #ifndef VIRTIO_BLK_NO_LEGACY 201 FEATURE_ENTRY(VIRTIO_BLK_F_BARRIER, \ 202 "VIRTIO_BLK_F_BARRIER: Request barriers supported"), 203 FEATURE_ENTRY(VIRTIO_BLK_F_SCSI, \ 204 "VIRTIO_BLK_F_SCSI: SCSI packet commands supported"), 205 FEATURE_ENTRY(VIRTIO_BLK_F_FLUSH, \ 206 "VIRTIO_BLK_F_FLUSH: Flush command supported"), 207 FEATURE_ENTRY(VIRTIO_BLK_F_CONFIG_WCE, \ 208 "VIRTIO_BLK_F_CONFIG_WCE: Cache writeback and writethrough modes " 209 "supported"), 210 #endif /* !VIRTIO_BLK_NO_LEGACY */ 211 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 212 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 213 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 214 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 215 "negotiation supported"), 216 { -1, "" } 217 }; 218 219 /* virtio-serial features mapping */ 220 qmp_virtio_feature_map_t virtio_serial_feature_map[] = { 221 FEATURE_ENTRY(VIRTIO_CONSOLE_F_SIZE, \ 222 "VIRTIO_CONSOLE_F_SIZE: Host providing console size"), 223 FEATURE_ENTRY(VIRTIO_CONSOLE_F_MULTIPORT, \ 224 "VIRTIO_CONSOLE_F_MULTIPORT: Multiple ports for device supported"), 225 FEATURE_ENTRY(VIRTIO_CONSOLE_F_EMERG_WRITE, \ 226 "VIRTIO_CONSOLE_F_EMERG_WRITE: Emergency write supported"), 227 { -1, "" } 228 }; 229 230 /* virtio-gpu features mapping */ 231 qmp_virtio_feature_map_t virtio_gpu_feature_map[] = { 232 FEATURE_ENTRY(VIRTIO_GPU_F_VIRGL, \ 233 "VIRTIO_GPU_F_VIRGL: Virgl 3D mode supported"), 234 FEATURE_ENTRY(VIRTIO_GPU_F_EDID, \ 235 "VIRTIO_GPU_F_EDID: EDID metadata supported"), 236 FEATURE_ENTRY(VIRTIO_GPU_F_RESOURCE_UUID, \ 237 "VIRTIO_GPU_F_RESOURCE_UUID: Resource UUID assigning supported"), 238 FEATURE_ENTRY(VIRTIO_GPU_F_RESOURCE_BLOB, \ 239 "VIRTIO_GPU_F_RESOURCE_BLOB: Size-based blob resources supported"), 240 FEATURE_ENTRY(VIRTIO_GPU_F_CONTEXT_INIT, \ 241 "VIRTIO_GPU_F_CONTEXT_INIT: Context types and synchronization " 242 "timelines supported"), 243 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 244 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 245 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 246 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 247 "negotiation supported"), 248 { -1, "" } 249 }; 250 251 /* virtio-input features mapping */ 252 qmp_virtio_feature_map_t virtio_input_feature_map[] = { 253 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 254 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 255 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 256 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 257 "negotiation supported"), 258 { -1, "" } 259 }; 260 261 /* virtio-net features mapping */ 262 qmp_virtio_feature_map_t virtio_net_feature_map[] = { 263 FEATURE_ENTRY(VIRTIO_NET_F_CSUM, \ 264 "VIRTIO_NET_F_CSUM: Device handling packets with partial checksum " 265 "supported"), 266 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_CSUM, \ 267 "VIRTIO_NET_F_GUEST_CSUM: Driver handling packets with partial " 268 "checksum supported"), 269 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \ 270 "VIRTIO_NET_F_CTRL_GUEST_OFFLOADS: Control channel offloading " 271 "reconfig. supported"), 272 FEATURE_ENTRY(VIRTIO_NET_F_MTU, \ 273 "VIRTIO_NET_F_MTU: Device max MTU reporting supported"), 274 FEATURE_ENTRY(VIRTIO_NET_F_MAC, \ 275 "VIRTIO_NET_F_MAC: Device has given MAC address"), 276 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_TSO4, \ 277 "VIRTIO_NET_F_GUEST_TSO4: Driver can receive TSOv4"), 278 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_TSO6, \ 279 "VIRTIO_NET_F_GUEST_TSO6: Driver can receive TSOv6"), 280 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_ECN, \ 281 "VIRTIO_NET_F_GUEST_ECN: Driver can receive TSO with ECN"), 282 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_UFO, \ 283 "VIRTIO_NET_F_GUEST_UFO: Driver can receive UFO"), 284 FEATURE_ENTRY(VIRTIO_NET_F_HOST_TSO4, \ 285 "VIRTIO_NET_F_HOST_TSO4: Device can receive TSOv4"), 286 FEATURE_ENTRY(VIRTIO_NET_F_HOST_TSO6, \ 287 "VIRTIO_NET_F_HOST_TSO6: Device can receive TSOv6"), 288 FEATURE_ENTRY(VIRTIO_NET_F_HOST_ECN, \ 289 "VIRTIO_NET_F_HOST_ECN: Device can receive TSO with ECN"), 290 FEATURE_ENTRY(VIRTIO_NET_F_HOST_UFO, \ 291 "VIRTIO_NET_F_HOST_UFO: Device can receive UFO"), 292 FEATURE_ENTRY(VIRTIO_NET_F_MRG_RXBUF, \ 293 "VIRTIO_NET_F_MRG_RXBUF: Driver can merge receive buffers"), 294 FEATURE_ENTRY(VIRTIO_NET_F_STATUS, \ 295 "VIRTIO_NET_F_STATUS: Configuration status field available"), 296 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_VQ, \ 297 "VIRTIO_NET_F_CTRL_VQ: Control channel available"), 298 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_RX, \ 299 "VIRTIO_NET_F_CTRL_RX: Control channel RX mode supported"), 300 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_VLAN, \ 301 "VIRTIO_NET_F_CTRL_VLAN: Control channel VLAN filtering supported"), 302 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_RX_EXTRA, \ 303 "VIRTIO_NET_F_CTRL_RX_EXTRA: Extra RX mode control supported"), 304 FEATURE_ENTRY(VIRTIO_NET_F_GUEST_ANNOUNCE, \ 305 "VIRTIO_NET_F_GUEST_ANNOUNCE: Driver sending gratuitous packets " 306 "supported"), 307 FEATURE_ENTRY(VIRTIO_NET_F_MQ, \ 308 "VIRTIO_NET_F_MQ: Multiqueue with automatic receive steering " 309 "supported"), 310 FEATURE_ENTRY(VIRTIO_NET_F_CTRL_MAC_ADDR, \ 311 "VIRTIO_NET_F_CTRL_MAC_ADDR: MAC address set through control " 312 "channel"), 313 FEATURE_ENTRY(VIRTIO_NET_F_HASH_REPORT, \ 314 "VIRTIO_NET_F_HASH_REPORT: Hash reporting supported"), 315 FEATURE_ENTRY(VIRTIO_NET_F_RSS, \ 316 "VIRTIO_NET_F_RSS: RSS RX steering supported"), 317 FEATURE_ENTRY(VIRTIO_NET_F_RSC_EXT, \ 318 "VIRTIO_NET_F_RSC_EXT: Extended coalescing info supported"), 319 FEATURE_ENTRY(VIRTIO_NET_F_STANDBY, \ 320 "VIRTIO_NET_F_STANDBY: Device acting as standby for primary " 321 "device with same MAC addr. supported"), 322 FEATURE_ENTRY(VIRTIO_NET_F_SPEED_DUPLEX, \ 323 "VIRTIO_NET_F_SPEED_DUPLEX: Device set linkspeed and duplex"), 324 #ifndef VIRTIO_NET_NO_LEGACY 325 FEATURE_ENTRY(VIRTIO_NET_F_GSO, \ 326 "VIRTIO_NET_F_GSO: Handling GSO-type packets supported"), 327 #endif /* !VIRTIO_NET_NO_LEGACY */ 328 FEATURE_ENTRY(VHOST_NET_F_VIRTIO_NET_HDR, \ 329 "VHOST_NET_F_VIRTIO_NET_HDR: Virtio-net headers for RX and TX " 330 "packets supported"), 331 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 332 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 333 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 334 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 335 "negotiation supported"), 336 { -1, "" } 337 }; 338 339 /* virtio-scsi features mapping */ 340 qmp_virtio_feature_map_t virtio_scsi_feature_map[] = { 341 FEATURE_ENTRY(VIRTIO_SCSI_F_INOUT, \ 342 "VIRTIO_SCSI_F_INOUT: Requests including read and writable data " 343 "buffers suppoted"), 344 FEATURE_ENTRY(VIRTIO_SCSI_F_HOTPLUG, \ 345 "VIRTIO_SCSI_F_HOTPLUG: Reporting and handling hot-plug events " 346 "supported"), 347 FEATURE_ENTRY(VIRTIO_SCSI_F_CHANGE, \ 348 "VIRTIO_SCSI_F_CHANGE: Reporting and handling LUN changes " 349 "supported"), 350 FEATURE_ENTRY(VIRTIO_SCSI_F_T10_PI, \ 351 "VIRTIO_SCSI_F_T10_PI: T10 info included in request header"), 352 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 353 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 354 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 355 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 356 "negotiation supported"), 357 { -1, "" } 358 }; 359 360 /* virtio/vhost-user-fs features mapping */ 361 qmp_virtio_feature_map_t virtio_fs_feature_map[] = { 362 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 363 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 364 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 365 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 366 "negotiation supported"), 367 { -1, "" } 368 }; 369 370 /* virtio/vhost-user-i2c features mapping */ 371 qmp_virtio_feature_map_t virtio_i2c_feature_map[] = { 372 FEATURE_ENTRY(VIRTIO_I2C_F_ZERO_LENGTH_REQUEST, \ 373 "VIRTIO_I2C_F_ZERO_LEGNTH_REQUEST: Zero length requests supported"), 374 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 375 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 376 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 377 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 378 "negotiation supported"), 379 { -1, "" } 380 }; 381 382 /* virtio/vhost-vsock features mapping */ 383 qmp_virtio_feature_map_t virtio_vsock_feature_map[] = { 384 FEATURE_ENTRY(VIRTIO_VSOCK_F_SEQPACKET, \ 385 "VIRTIO_VSOCK_F_SEQPACKET: SOCK_SEQPACKET supported"), 386 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 387 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 388 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 389 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 390 "negotiation supported"), 391 { -1, "" } 392 }; 393 394 /* virtio-balloon features mapping */ 395 qmp_virtio_feature_map_t virtio_balloon_feature_map[] = { 396 FEATURE_ENTRY(VIRTIO_BALLOON_F_MUST_TELL_HOST, \ 397 "VIRTIO_BALLOON_F_MUST_TELL_HOST: Tell host before reclaiming " 398 "pages"), 399 FEATURE_ENTRY(VIRTIO_BALLOON_F_STATS_VQ, \ 400 "VIRTIO_BALLOON_F_STATS_VQ: Guest memory stats VQ available"), 401 FEATURE_ENTRY(VIRTIO_BALLOON_F_DEFLATE_ON_OOM, \ 402 "VIRTIO_BALLOON_F_DEFLATE_ON_OOM: Deflate balloon when guest OOM"), 403 FEATURE_ENTRY(VIRTIO_BALLOON_F_FREE_PAGE_HINT, \ 404 "VIRTIO_BALLOON_F_FREE_PAGE_HINT: VQ reporting free pages enabled"), 405 FEATURE_ENTRY(VIRTIO_BALLOON_F_PAGE_POISON, \ 406 "VIRTIO_BALLOON_F_PAGE_POISON: Guest page poisoning enabled"), 407 FEATURE_ENTRY(VIRTIO_BALLOON_F_REPORTING, \ 408 "VIRTIO_BALLOON_F_REPORTING: Page reporting VQ enabled"), 409 { -1, "" } 410 }; 411 412 /* virtio-crypto features mapping */ 413 qmp_virtio_feature_map_t virtio_crypto_feature_map[] = { 414 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 415 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 416 { -1, "" } 417 }; 418 419 /* virtio-iommu features mapping */ 420 qmp_virtio_feature_map_t virtio_iommu_feature_map[] = { 421 FEATURE_ENTRY(VIRTIO_IOMMU_F_INPUT_RANGE, \ 422 "VIRTIO_IOMMU_F_INPUT_RANGE: Range of available virtual addrs. " 423 "available"), 424 FEATURE_ENTRY(VIRTIO_IOMMU_F_DOMAIN_RANGE, \ 425 "VIRTIO_IOMMU_F_DOMAIN_RANGE: Number of supported domains " 426 "available"), 427 FEATURE_ENTRY(VIRTIO_IOMMU_F_MAP_UNMAP, \ 428 "VIRTIO_IOMMU_F_MAP_UNMAP: Map and unmap requests available"), 429 FEATURE_ENTRY(VIRTIO_IOMMU_F_BYPASS, \ 430 "VIRTIO_IOMMU_F_BYPASS: Endpoints not attached to domains are in " 431 "bypass mode"), 432 FEATURE_ENTRY(VIRTIO_IOMMU_F_PROBE, \ 433 "VIRTIO_IOMMU_F_PROBE: Probe requests available"), 434 FEATURE_ENTRY(VIRTIO_IOMMU_F_MMIO, \ 435 "VIRTIO_IOMMU_F_MMIO: VIRTIO_IOMMU_MAP_F_MMIO flag available"), 436 FEATURE_ENTRY(VIRTIO_IOMMU_F_BYPASS_CONFIG, \ 437 "VIRTIO_IOMMU_F_BYPASS_CONFIG: Bypass field of IOMMU config " 438 "available"), 439 { -1, "" } 440 }; 441 442 /* virtio-mem features mapping */ 443 qmp_virtio_feature_map_t virtio_mem_feature_map[] = { 444 #ifndef CONFIG_ACPI 445 FEATURE_ENTRY(VIRTIO_MEM_F_ACPI_PXM, \ 446 "VIRTIO_MEM_F_ACPI_PXM: node_id is an ACPI PXM and is valid"), 447 #endif /* !CONFIG_ACPI */ 448 FEATURE_ENTRY(VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE, \ 449 "VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE: Unplugged memory cannot be " 450 "accessed"), 451 { -1, "" } 452 }; 453 454 /* virtio-rng features mapping */ 455 qmp_virtio_feature_map_t virtio_rng_feature_map[] = { 456 FEATURE_ENTRY(VHOST_F_LOG_ALL, \ 457 "VHOST_F_LOG_ALL: Logging write descriptors supported"), 458 FEATURE_ENTRY(VHOST_USER_F_PROTOCOL_FEATURES, \ 459 "VHOST_USER_F_PROTOCOL_FEATURES: Vhost-user protocol features " 460 "negotiation supported"), 461 { -1, "" } 462 }; 463 464 /* 465 * The alignment to use between consumer and producer parts of vring. 466 * x86 pagesize again. This is the default, used by transports like PCI 467 * which don't provide a means for the guest to tell the host the alignment. 468 */ 469 #define VIRTIO_PCI_VRING_ALIGN 4096 470 471 typedef struct VRingDesc 472 { 473 uint64_t addr; 474 uint32_t len; 475 uint16_t flags; 476 uint16_t next; 477 } VRingDesc; 478 479 typedef struct VRingPackedDesc { 480 uint64_t addr; 481 uint32_t len; 482 uint16_t id; 483 uint16_t flags; 484 } VRingPackedDesc; 485 486 typedef struct VRingAvail 487 { 488 uint16_t flags; 489 uint16_t idx; 490 uint16_t ring[]; 491 } VRingAvail; 492 493 typedef struct VRingUsedElem 494 { 495 uint32_t id; 496 uint32_t len; 497 } VRingUsedElem; 498 499 typedef struct VRingUsed 500 { 501 uint16_t flags; 502 uint16_t idx; 503 VRingUsedElem ring[]; 504 } VRingUsed; 505 506 typedef struct VRingMemoryRegionCaches { 507 struct rcu_head rcu; 508 MemoryRegionCache desc; 509 MemoryRegionCache avail; 510 MemoryRegionCache used; 511 } VRingMemoryRegionCaches; 512 513 typedef struct VRing 514 { 515 unsigned int num; 516 unsigned int num_default; 517 unsigned int align; 518 hwaddr desc; 519 hwaddr avail; 520 hwaddr used; 521 VRingMemoryRegionCaches *caches; 522 } VRing; 523 524 typedef struct VRingPackedDescEvent { 525 uint16_t off_wrap; 526 uint16_t flags; 527 } VRingPackedDescEvent ; 528 529 struct VirtQueue 530 { 531 VRing vring; 532 VirtQueueElement *used_elems; 533 534 /* Next head to pop */ 535 uint16_t last_avail_idx; 536 bool last_avail_wrap_counter; 537 538 /* Last avail_idx read from VQ. */ 539 uint16_t shadow_avail_idx; 540 bool shadow_avail_wrap_counter; 541 542 uint16_t used_idx; 543 bool used_wrap_counter; 544 545 /* Last used index value we have signalled on */ 546 uint16_t signalled_used; 547 548 /* Last used index value we have signalled on */ 549 bool signalled_used_valid; 550 551 /* Notification enabled? */ 552 bool notification; 553 554 uint16_t queue_index; 555 556 unsigned int inuse; 557 558 uint16_t vector; 559 VirtIOHandleOutput handle_output; 560 VirtIODevice *vdev; 561 EventNotifier guest_notifier; 562 EventNotifier host_notifier; 563 bool host_notifier_enabled; 564 QLIST_ENTRY(VirtQueue) node; 565 }; 566 567 const char *virtio_device_names[] = { 568 [VIRTIO_ID_NET] = "virtio-net", 569 [VIRTIO_ID_BLOCK] = "virtio-blk", 570 [VIRTIO_ID_CONSOLE] = "virtio-serial", 571 [VIRTIO_ID_RNG] = "virtio-rng", 572 [VIRTIO_ID_BALLOON] = "virtio-balloon", 573 [VIRTIO_ID_IOMEM] = "virtio-iomem", 574 [VIRTIO_ID_RPMSG] = "virtio-rpmsg", 575 [VIRTIO_ID_SCSI] = "virtio-scsi", 576 [VIRTIO_ID_9P] = "virtio-9p", 577 [VIRTIO_ID_MAC80211_WLAN] = "virtio-mac-wlan", 578 [VIRTIO_ID_RPROC_SERIAL] = "virtio-rproc-serial", 579 [VIRTIO_ID_CAIF] = "virtio-caif", 580 [VIRTIO_ID_MEMORY_BALLOON] = "virtio-mem-balloon", 581 [VIRTIO_ID_GPU] = "virtio-gpu", 582 [VIRTIO_ID_CLOCK] = "virtio-clk", 583 [VIRTIO_ID_INPUT] = "virtio-input", 584 [VIRTIO_ID_VSOCK] = "vhost-vsock", 585 [VIRTIO_ID_CRYPTO] = "virtio-crypto", 586 [VIRTIO_ID_SIGNAL_DIST] = "virtio-signal", 587 [VIRTIO_ID_PSTORE] = "virtio-pstore", 588 [VIRTIO_ID_IOMMU] = "virtio-iommu", 589 [VIRTIO_ID_MEM] = "virtio-mem", 590 [VIRTIO_ID_SOUND] = "virtio-sound", 591 [VIRTIO_ID_FS] = "virtio-user-fs", 592 [VIRTIO_ID_PMEM] = "virtio-pmem", 593 [VIRTIO_ID_RPMB] = "virtio-rpmb", 594 [VIRTIO_ID_MAC80211_HWSIM] = "virtio-mac-hwsim", 595 [VIRTIO_ID_VIDEO_ENCODER] = "virtio-vid-encoder", 596 [VIRTIO_ID_VIDEO_DECODER] = "virtio-vid-decoder", 597 [VIRTIO_ID_SCMI] = "virtio-scmi", 598 [VIRTIO_ID_NITRO_SEC_MOD] = "virtio-nitro-sec-mod", 599 [VIRTIO_ID_I2C_ADAPTER] = "vhost-user-i2c", 600 [VIRTIO_ID_WATCHDOG] = "virtio-watchdog", 601 [VIRTIO_ID_CAN] = "virtio-can", 602 [VIRTIO_ID_DMABUF] = "virtio-dmabuf", 603 [VIRTIO_ID_PARAM_SERV] = "virtio-param-serv", 604 [VIRTIO_ID_AUDIO_POLICY] = "virtio-audio-pol", 605 [VIRTIO_ID_BT] = "virtio-bluetooth", 606 [VIRTIO_ID_GPIO] = "virtio-gpio" 607 }; 608 609 static const char *virtio_id_to_name(uint16_t device_id) 610 { 611 assert(device_id < G_N_ELEMENTS(virtio_device_names)); 612 const char *name = virtio_device_names[device_id]; 613 assert(name != NULL); 614 return name; 615 } 616 617 /* Called within call_rcu(). */ 618 static void virtio_free_region_cache(VRingMemoryRegionCaches *caches) 619 { 620 assert(caches != NULL); 621 address_space_cache_destroy(&caches->desc); 622 address_space_cache_destroy(&caches->avail); 623 address_space_cache_destroy(&caches->used); 624 g_free(caches); 625 } 626 627 static void virtio_virtqueue_reset_region_cache(struct VirtQueue *vq) 628 { 629 VRingMemoryRegionCaches *caches; 630 631 caches = qatomic_read(&vq->vring.caches); 632 qatomic_rcu_set(&vq->vring.caches, NULL); 633 if (caches) { 634 call_rcu(caches, virtio_free_region_cache, rcu); 635 } 636 } 637 638 static void virtio_init_region_cache(VirtIODevice *vdev, int n) 639 { 640 VirtQueue *vq = &vdev->vq[n]; 641 VRingMemoryRegionCaches *old = vq->vring.caches; 642 VRingMemoryRegionCaches *new = NULL; 643 hwaddr addr, size; 644 int64_t len; 645 bool packed; 646 647 648 addr = vq->vring.desc; 649 if (!addr) { 650 goto out_no_cache; 651 } 652 new = g_new0(VRingMemoryRegionCaches, 1); 653 size = virtio_queue_get_desc_size(vdev, n); 654 packed = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ? 655 true : false; 656 len = address_space_cache_init(&new->desc, vdev->dma_as, 657 addr, size, packed); 658 if (len < size) { 659 virtio_error(vdev, "Cannot map desc"); 660 goto err_desc; 661 } 662 663 size = virtio_queue_get_used_size(vdev, n); 664 len = address_space_cache_init(&new->used, vdev->dma_as, 665 vq->vring.used, size, true); 666 if (len < size) { 667 virtio_error(vdev, "Cannot map used"); 668 goto err_used; 669 } 670 671 size = virtio_queue_get_avail_size(vdev, n); 672 len = address_space_cache_init(&new->avail, vdev->dma_as, 673 vq->vring.avail, size, false); 674 if (len < size) { 675 virtio_error(vdev, "Cannot map avail"); 676 goto err_avail; 677 } 678 679 qatomic_rcu_set(&vq->vring.caches, new); 680 if (old) { 681 call_rcu(old, virtio_free_region_cache, rcu); 682 } 683 return; 684 685 err_avail: 686 address_space_cache_destroy(&new->avail); 687 err_used: 688 address_space_cache_destroy(&new->used); 689 err_desc: 690 address_space_cache_destroy(&new->desc); 691 out_no_cache: 692 g_free(new); 693 virtio_virtqueue_reset_region_cache(vq); 694 } 695 696 /* virt queue functions */ 697 void virtio_queue_update_rings(VirtIODevice *vdev, int n) 698 { 699 VRing *vring = &vdev->vq[n].vring; 700 701 if (!vring->num || !vring->desc || !vring->align) { 702 /* not yet setup -> nothing to do */ 703 return; 704 } 705 vring->avail = vring->desc + vring->num * sizeof(VRingDesc); 706 vring->used = vring_align(vring->avail + 707 offsetof(VRingAvail, ring[vring->num]), 708 vring->align); 709 virtio_init_region_cache(vdev, n); 710 } 711 712 /* Called within rcu_read_lock(). */ 713 static void vring_split_desc_read(VirtIODevice *vdev, VRingDesc *desc, 714 MemoryRegionCache *cache, int i) 715 { 716 address_space_read_cached(cache, i * sizeof(VRingDesc), 717 desc, sizeof(VRingDesc)); 718 virtio_tswap64s(vdev, &desc->addr); 719 virtio_tswap32s(vdev, &desc->len); 720 virtio_tswap16s(vdev, &desc->flags); 721 virtio_tswap16s(vdev, &desc->next); 722 } 723 724 static void vring_packed_event_read(VirtIODevice *vdev, 725 MemoryRegionCache *cache, 726 VRingPackedDescEvent *e) 727 { 728 hwaddr off_off = offsetof(VRingPackedDescEvent, off_wrap); 729 hwaddr off_flags = offsetof(VRingPackedDescEvent, flags); 730 731 e->flags = virtio_lduw_phys_cached(vdev, cache, off_flags); 732 /* Make sure flags is seen before off_wrap */ 733 smp_rmb(); 734 e->off_wrap = virtio_lduw_phys_cached(vdev, cache, off_off); 735 virtio_tswap16s(vdev, &e->flags); 736 } 737 738 static void vring_packed_off_wrap_write(VirtIODevice *vdev, 739 MemoryRegionCache *cache, 740 uint16_t off_wrap) 741 { 742 hwaddr off = offsetof(VRingPackedDescEvent, off_wrap); 743 744 virtio_stw_phys_cached(vdev, cache, off, off_wrap); 745 address_space_cache_invalidate(cache, off, sizeof(off_wrap)); 746 } 747 748 static void vring_packed_flags_write(VirtIODevice *vdev, 749 MemoryRegionCache *cache, uint16_t flags) 750 { 751 hwaddr off = offsetof(VRingPackedDescEvent, flags); 752 753 virtio_stw_phys_cached(vdev, cache, off, flags); 754 address_space_cache_invalidate(cache, off, sizeof(flags)); 755 } 756 757 /* Called within rcu_read_lock(). */ 758 static VRingMemoryRegionCaches *vring_get_region_caches(struct VirtQueue *vq) 759 { 760 return qatomic_rcu_read(&vq->vring.caches); 761 } 762 763 /* Called within rcu_read_lock(). */ 764 static inline uint16_t vring_avail_flags(VirtQueue *vq) 765 { 766 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 767 hwaddr pa = offsetof(VRingAvail, flags); 768 769 if (!caches) { 770 return 0; 771 } 772 773 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); 774 } 775 776 /* Called within rcu_read_lock(). */ 777 static inline uint16_t vring_avail_idx(VirtQueue *vq) 778 { 779 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 780 hwaddr pa = offsetof(VRingAvail, idx); 781 782 if (!caches) { 783 return 0; 784 } 785 786 vq->shadow_avail_idx = virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); 787 return vq->shadow_avail_idx; 788 } 789 790 /* Called within rcu_read_lock(). */ 791 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i) 792 { 793 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 794 hwaddr pa = offsetof(VRingAvail, ring[i]); 795 796 if (!caches) { 797 return 0; 798 } 799 800 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); 801 } 802 803 /* Called within rcu_read_lock(). */ 804 static inline uint16_t vring_get_used_event(VirtQueue *vq) 805 { 806 return vring_avail_ring(vq, vq->vring.num); 807 } 808 809 /* Called within rcu_read_lock(). */ 810 static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem, 811 int i) 812 { 813 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 814 hwaddr pa = offsetof(VRingUsed, ring[i]); 815 816 if (!caches) { 817 return; 818 } 819 820 virtio_tswap32s(vq->vdev, &uelem->id); 821 virtio_tswap32s(vq->vdev, &uelem->len); 822 address_space_write_cached(&caches->used, pa, uelem, sizeof(VRingUsedElem)); 823 address_space_cache_invalidate(&caches->used, pa, sizeof(VRingUsedElem)); 824 } 825 826 /* Called within rcu_read_lock(). */ 827 static inline uint16_t vring_used_flags(VirtQueue *vq) 828 { 829 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 830 hwaddr pa = offsetof(VRingUsed, flags); 831 832 if (!caches) { 833 return 0; 834 } 835 836 return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); 837 } 838 839 /* Called within rcu_read_lock(). */ 840 static uint16_t vring_used_idx(VirtQueue *vq) 841 { 842 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 843 hwaddr pa = offsetof(VRingUsed, idx); 844 845 if (!caches) { 846 return 0; 847 } 848 849 return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); 850 } 851 852 /* Called within rcu_read_lock(). */ 853 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val) 854 { 855 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 856 hwaddr pa = offsetof(VRingUsed, idx); 857 858 if (caches) { 859 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val); 860 address_space_cache_invalidate(&caches->used, pa, sizeof(val)); 861 } 862 863 vq->used_idx = val; 864 } 865 866 /* Called within rcu_read_lock(). */ 867 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask) 868 { 869 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 870 VirtIODevice *vdev = vq->vdev; 871 hwaddr pa = offsetof(VRingUsed, flags); 872 uint16_t flags; 873 874 if (!caches) { 875 return; 876 } 877 878 flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); 879 virtio_stw_phys_cached(vdev, &caches->used, pa, flags | mask); 880 address_space_cache_invalidate(&caches->used, pa, sizeof(flags)); 881 } 882 883 /* Called within rcu_read_lock(). */ 884 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask) 885 { 886 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 887 VirtIODevice *vdev = vq->vdev; 888 hwaddr pa = offsetof(VRingUsed, flags); 889 uint16_t flags; 890 891 if (!caches) { 892 return; 893 } 894 895 flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); 896 virtio_stw_phys_cached(vdev, &caches->used, pa, flags & ~mask); 897 address_space_cache_invalidate(&caches->used, pa, sizeof(flags)); 898 } 899 900 /* Called within rcu_read_lock(). */ 901 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val) 902 { 903 VRingMemoryRegionCaches *caches; 904 hwaddr pa; 905 if (!vq->notification) { 906 return; 907 } 908 909 caches = vring_get_region_caches(vq); 910 if (!caches) { 911 return; 912 } 913 914 pa = offsetof(VRingUsed, ring[vq->vring.num]); 915 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val); 916 address_space_cache_invalidate(&caches->used, pa, sizeof(val)); 917 } 918 919 static void virtio_queue_split_set_notification(VirtQueue *vq, int enable) 920 { 921 RCU_READ_LOCK_GUARD(); 922 923 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { 924 vring_set_avail_event(vq, vring_avail_idx(vq)); 925 } else if (enable) { 926 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY); 927 } else { 928 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY); 929 } 930 if (enable) { 931 /* Expose avail event/used flags before caller checks the avail idx. */ 932 smp_mb(); 933 } 934 } 935 936 static void virtio_queue_packed_set_notification(VirtQueue *vq, int enable) 937 { 938 uint16_t off_wrap; 939 VRingPackedDescEvent e; 940 VRingMemoryRegionCaches *caches; 941 942 RCU_READ_LOCK_GUARD(); 943 caches = vring_get_region_caches(vq); 944 if (!caches) { 945 return; 946 } 947 948 vring_packed_event_read(vq->vdev, &caches->used, &e); 949 950 if (!enable) { 951 e.flags = VRING_PACKED_EVENT_FLAG_DISABLE; 952 } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { 953 off_wrap = vq->shadow_avail_idx | vq->shadow_avail_wrap_counter << 15; 954 vring_packed_off_wrap_write(vq->vdev, &caches->used, off_wrap); 955 /* Make sure off_wrap is wrote before flags */ 956 smp_wmb(); 957 e.flags = VRING_PACKED_EVENT_FLAG_DESC; 958 } else { 959 e.flags = VRING_PACKED_EVENT_FLAG_ENABLE; 960 } 961 962 vring_packed_flags_write(vq->vdev, &caches->used, e.flags); 963 if (enable) { 964 /* Expose avail event/used flags before caller checks the avail idx. */ 965 smp_mb(); 966 } 967 } 968 969 bool virtio_queue_get_notification(VirtQueue *vq) 970 { 971 return vq->notification; 972 } 973 974 void virtio_queue_set_notification(VirtQueue *vq, int enable) 975 { 976 vq->notification = enable; 977 978 if (!vq->vring.desc) { 979 return; 980 } 981 982 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 983 virtio_queue_packed_set_notification(vq, enable); 984 } else { 985 virtio_queue_split_set_notification(vq, enable); 986 } 987 } 988 989 int virtio_queue_ready(VirtQueue *vq) 990 { 991 return vq->vring.avail != 0; 992 } 993 994 static void vring_packed_desc_read_flags(VirtIODevice *vdev, 995 uint16_t *flags, 996 MemoryRegionCache *cache, 997 int i) 998 { 999 hwaddr off = i * sizeof(VRingPackedDesc) + offsetof(VRingPackedDesc, flags); 1000 1001 *flags = virtio_lduw_phys_cached(vdev, cache, off); 1002 } 1003 1004 static void vring_packed_desc_read(VirtIODevice *vdev, 1005 VRingPackedDesc *desc, 1006 MemoryRegionCache *cache, 1007 int i, bool strict_order) 1008 { 1009 hwaddr off = i * sizeof(VRingPackedDesc); 1010 1011 vring_packed_desc_read_flags(vdev, &desc->flags, cache, i); 1012 1013 if (strict_order) { 1014 /* Make sure flags is read before the rest fields. */ 1015 smp_rmb(); 1016 } 1017 1018 address_space_read_cached(cache, off + offsetof(VRingPackedDesc, addr), 1019 &desc->addr, sizeof(desc->addr)); 1020 address_space_read_cached(cache, off + offsetof(VRingPackedDesc, id), 1021 &desc->id, sizeof(desc->id)); 1022 address_space_read_cached(cache, off + offsetof(VRingPackedDesc, len), 1023 &desc->len, sizeof(desc->len)); 1024 virtio_tswap64s(vdev, &desc->addr); 1025 virtio_tswap16s(vdev, &desc->id); 1026 virtio_tswap32s(vdev, &desc->len); 1027 } 1028 1029 static void vring_packed_desc_write_data(VirtIODevice *vdev, 1030 VRingPackedDesc *desc, 1031 MemoryRegionCache *cache, 1032 int i) 1033 { 1034 hwaddr off_id = i * sizeof(VRingPackedDesc) + 1035 offsetof(VRingPackedDesc, id); 1036 hwaddr off_len = i * sizeof(VRingPackedDesc) + 1037 offsetof(VRingPackedDesc, len); 1038 1039 virtio_tswap32s(vdev, &desc->len); 1040 virtio_tswap16s(vdev, &desc->id); 1041 address_space_write_cached(cache, off_id, &desc->id, sizeof(desc->id)); 1042 address_space_cache_invalidate(cache, off_id, sizeof(desc->id)); 1043 address_space_write_cached(cache, off_len, &desc->len, sizeof(desc->len)); 1044 address_space_cache_invalidate(cache, off_len, sizeof(desc->len)); 1045 } 1046 1047 static void vring_packed_desc_write_flags(VirtIODevice *vdev, 1048 VRingPackedDesc *desc, 1049 MemoryRegionCache *cache, 1050 int i) 1051 { 1052 hwaddr off = i * sizeof(VRingPackedDesc) + offsetof(VRingPackedDesc, flags); 1053 1054 virtio_stw_phys_cached(vdev, cache, off, desc->flags); 1055 address_space_cache_invalidate(cache, off, sizeof(desc->flags)); 1056 } 1057 1058 static void vring_packed_desc_write(VirtIODevice *vdev, 1059 VRingPackedDesc *desc, 1060 MemoryRegionCache *cache, 1061 int i, bool strict_order) 1062 { 1063 vring_packed_desc_write_data(vdev, desc, cache, i); 1064 if (strict_order) { 1065 /* Make sure data is wrote before flags. */ 1066 smp_wmb(); 1067 } 1068 vring_packed_desc_write_flags(vdev, desc, cache, i); 1069 } 1070 1071 static inline bool is_desc_avail(uint16_t flags, bool wrap_counter) 1072 { 1073 bool avail, used; 1074 1075 avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL)); 1076 used = !!(flags & (1 << VRING_PACKED_DESC_F_USED)); 1077 return (avail != used) && (avail == wrap_counter); 1078 } 1079 1080 /* Fetch avail_idx from VQ memory only when we really need to know if 1081 * guest has added some buffers. 1082 * Called within rcu_read_lock(). */ 1083 static int virtio_queue_empty_rcu(VirtQueue *vq) 1084 { 1085 if (virtio_device_disabled(vq->vdev)) { 1086 return 1; 1087 } 1088 1089 if (unlikely(!vq->vring.avail)) { 1090 return 1; 1091 } 1092 1093 if (vq->shadow_avail_idx != vq->last_avail_idx) { 1094 return 0; 1095 } 1096 1097 return vring_avail_idx(vq) == vq->last_avail_idx; 1098 } 1099 1100 static int virtio_queue_split_empty(VirtQueue *vq) 1101 { 1102 bool empty; 1103 1104 if (virtio_device_disabled(vq->vdev)) { 1105 return 1; 1106 } 1107 1108 if (unlikely(!vq->vring.avail)) { 1109 return 1; 1110 } 1111 1112 if (vq->shadow_avail_idx != vq->last_avail_idx) { 1113 return 0; 1114 } 1115 1116 RCU_READ_LOCK_GUARD(); 1117 empty = vring_avail_idx(vq) == vq->last_avail_idx; 1118 return empty; 1119 } 1120 1121 /* Called within rcu_read_lock(). */ 1122 static int virtio_queue_packed_empty_rcu(VirtQueue *vq) 1123 { 1124 struct VRingPackedDesc desc; 1125 VRingMemoryRegionCaches *cache; 1126 1127 if (unlikely(!vq->vring.desc)) { 1128 return 1; 1129 } 1130 1131 cache = vring_get_region_caches(vq); 1132 if (!cache) { 1133 return 1; 1134 } 1135 1136 vring_packed_desc_read_flags(vq->vdev, &desc.flags, &cache->desc, 1137 vq->last_avail_idx); 1138 1139 return !is_desc_avail(desc.flags, vq->last_avail_wrap_counter); 1140 } 1141 1142 static int virtio_queue_packed_empty(VirtQueue *vq) 1143 { 1144 RCU_READ_LOCK_GUARD(); 1145 return virtio_queue_packed_empty_rcu(vq); 1146 } 1147 1148 int virtio_queue_empty(VirtQueue *vq) 1149 { 1150 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1151 return virtio_queue_packed_empty(vq); 1152 } else { 1153 return virtio_queue_split_empty(vq); 1154 } 1155 } 1156 1157 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem, 1158 unsigned int len) 1159 { 1160 AddressSpace *dma_as = vq->vdev->dma_as; 1161 unsigned int offset; 1162 int i; 1163 1164 offset = 0; 1165 for (i = 0; i < elem->in_num; i++) { 1166 size_t size = MIN(len - offset, elem->in_sg[i].iov_len); 1167 1168 dma_memory_unmap(dma_as, elem->in_sg[i].iov_base, 1169 elem->in_sg[i].iov_len, 1170 DMA_DIRECTION_FROM_DEVICE, size); 1171 1172 offset += size; 1173 } 1174 1175 for (i = 0; i < elem->out_num; i++) 1176 dma_memory_unmap(dma_as, elem->out_sg[i].iov_base, 1177 elem->out_sg[i].iov_len, 1178 DMA_DIRECTION_TO_DEVICE, 1179 elem->out_sg[i].iov_len); 1180 } 1181 1182 /* virtqueue_detach_element: 1183 * @vq: The #VirtQueue 1184 * @elem: The #VirtQueueElement 1185 * @len: number of bytes written 1186 * 1187 * Detach the element from the virtqueue. This function is suitable for device 1188 * reset or other situations where a #VirtQueueElement is simply freed and will 1189 * not be pushed or discarded. 1190 */ 1191 void virtqueue_detach_element(VirtQueue *vq, const VirtQueueElement *elem, 1192 unsigned int len) 1193 { 1194 vq->inuse -= elem->ndescs; 1195 virtqueue_unmap_sg(vq, elem, len); 1196 } 1197 1198 static void virtqueue_split_rewind(VirtQueue *vq, unsigned int num) 1199 { 1200 vq->last_avail_idx -= num; 1201 } 1202 1203 static void virtqueue_packed_rewind(VirtQueue *vq, unsigned int num) 1204 { 1205 if (vq->last_avail_idx < num) { 1206 vq->last_avail_idx = vq->vring.num + vq->last_avail_idx - num; 1207 vq->last_avail_wrap_counter ^= 1; 1208 } else { 1209 vq->last_avail_idx -= num; 1210 } 1211 } 1212 1213 /* virtqueue_unpop: 1214 * @vq: The #VirtQueue 1215 * @elem: The #VirtQueueElement 1216 * @len: number of bytes written 1217 * 1218 * Pretend the most recent element wasn't popped from the virtqueue. The next 1219 * call to virtqueue_pop() will refetch the element. 1220 */ 1221 void virtqueue_unpop(VirtQueue *vq, const VirtQueueElement *elem, 1222 unsigned int len) 1223 { 1224 1225 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1226 virtqueue_packed_rewind(vq, 1); 1227 } else { 1228 virtqueue_split_rewind(vq, 1); 1229 } 1230 1231 virtqueue_detach_element(vq, elem, len); 1232 } 1233 1234 /* virtqueue_rewind: 1235 * @vq: The #VirtQueue 1236 * @num: Number of elements to push back 1237 * 1238 * Pretend that elements weren't popped from the virtqueue. The next 1239 * virtqueue_pop() will refetch the oldest element. 1240 * 1241 * Use virtqueue_unpop() instead if you have a VirtQueueElement. 1242 * 1243 * Returns: true on success, false if @num is greater than the number of in use 1244 * elements. 1245 */ 1246 bool virtqueue_rewind(VirtQueue *vq, unsigned int num) 1247 { 1248 if (num > vq->inuse) { 1249 return false; 1250 } 1251 1252 vq->inuse -= num; 1253 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1254 virtqueue_packed_rewind(vq, num); 1255 } else { 1256 virtqueue_split_rewind(vq, num); 1257 } 1258 return true; 1259 } 1260 1261 static void virtqueue_split_fill(VirtQueue *vq, const VirtQueueElement *elem, 1262 unsigned int len, unsigned int idx) 1263 { 1264 VRingUsedElem uelem; 1265 1266 if (unlikely(!vq->vring.used)) { 1267 return; 1268 } 1269 1270 idx = (idx + vq->used_idx) % vq->vring.num; 1271 1272 uelem.id = elem->index; 1273 uelem.len = len; 1274 vring_used_write(vq, &uelem, idx); 1275 } 1276 1277 static void virtqueue_packed_fill(VirtQueue *vq, const VirtQueueElement *elem, 1278 unsigned int len, unsigned int idx) 1279 { 1280 vq->used_elems[idx].index = elem->index; 1281 vq->used_elems[idx].len = len; 1282 vq->used_elems[idx].ndescs = elem->ndescs; 1283 } 1284 1285 static void virtqueue_packed_fill_desc(VirtQueue *vq, 1286 const VirtQueueElement *elem, 1287 unsigned int idx, 1288 bool strict_order) 1289 { 1290 uint16_t head; 1291 VRingMemoryRegionCaches *caches; 1292 VRingPackedDesc desc = { 1293 .id = elem->index, 1294 .len = elem->len, 1295 }; 1296 bool wrap_counter = vq->used_wrap_counter; 1297 1298 if (unlikely(!vq->vring.desc)) { 1299 return; 1300 } 1301 1302 head = vq->used_idx + idx; 1303 if (head >= vq->vring.num) { 1304 head -= vq->vring.num; 1305 wrap_counter ^= 1; 1306 } 1307 if (wrap_counter) { 1308 desc.flags |= (1 << VRING_PACKED_DESC_F_AVAIL); 1309 desc.flags |= (1 << VRING_PACKED_DESC_F_USED); 1310 } else { 1311 desc.flags &= ~(1 << VRING_PACKED_DESC_F_AVAIL); 1312 desc.flags &= ~(1 << VRING_PACKED_DESC_F_USED); 1313 } 1314 1315 caches = vring_get_region_caches(vq); 1316 if (!caches) { 1317 return; 1318 } 1319 1320 vring_packed_desc_write(vq->vdev, &desc, &caches->desc, head, strict_order); 1321 } 1322 1323 /* Called within rcu_read_lock(). */ 1324 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem, 1325 unsigned int len, unsigned int idx) 1326 { 1327 trace_virtqueue_fill(vq, elem, len, idx); 1328 1329 virtqueue_unmap_sg(vq, elem, len); 1330 1331 if (virtio_device_disabled(vq->vdev)) { 1332 return; 1333 } 1334 1335 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1336 virtqueue_packed_fill(vq, elem, len, idx); 1337 } else { 1338 virtqueue_split_fill(vq, elem, len, idx); 1339 } 1340 } 1341 1342 /* Called within rcu_read_lock(). */ 1343 static void virtqueue_split_flush(VirtQueue *vq, unsigned int count) 1344 { 1345 uint16_t old, new; 1346 1347 if (unlikely(!vq->vring.used)) { 1348 return; 1349 } 1350 1351 /* Make sure buffer is written before we update index. */ 1352 smp_wmb(); 1353 trace_virtqueue_flush(vq, count); 1354 old = vq->used_idx; 1355 new = old + count; 1356 vring_used_idx_set(vq, new); 1357 vq->inuse -= count; 1358 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old))) 1359 vq->signalled_used_valid = false; 1360 } 1361 1362 static void virtqueue_packed_flush(VirtQueue *vq, unsigned int count) 1363 { 1364 unsigned int i, ndescs = 0; 1365 1366 if (unlikely(!vq->vring.desc)) { 1367 return; 1368 } 1369 1370 for (i = 1; i < count; i++) { 1371 virtqueue_packed_fill_desc(vq, &vq->used_elems[i], i, false); 1372 ndescs += vq->used_elems[i].ndescs; 1373 } 1374 virtqueue_packed_fill_desc(vq, &vq->used_elems[0], 0, true); 1375 ndescs += vq->used_elems[0].ndescs; 1376 1377 vq->inuse -= ndescs; 1378 vq->used_idx += ndescs; 1379 if (vq->used_idx >= vq->vring.num) { 1380 vq->used_idx -= vq->vring.num; 1381 vq->used_wrap_counter ^= 1; 1382 vq->signalled_used_valid = false; 1383 } 1384 } 1385 1386 void virtqueue_flush(VirtQueue *vq, unsigned int count) 1387 { 1388 if (virtio_device_disabled(vq->vdev)) { 1389 vq->inuse -= count; 1390 return; 1391 } 1392 1393 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1394 virtqueue_packed_flush(vq, count); 1395 } else { 1396 virtqueue_split_flush(vq, count); 1397 } 1398 } 1399 1400 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem, 1401 unsigned int len) 1402 { 1403 RCU_READ_LOCK_GUARD(); 1404 virtqueue_fill(vq, elem, len, 0); 1405 virtqueue_flush(vq, 1); 1406 } 1407 1408 /* Called within rcu_read_lock(). */ 1409 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx) 1410 { 1411 uint16_t num_heads = vring_avail_idx(vq) - idx; 1412 1413 /* Check it isn't doing very strange things with descriptor numbers. */ 1414 if (num_heads > vq->vring.num) { 1415 virtio_error(vq->vdev, "Guest moved used index from %u to %u", 1416 idx, vq->shadow_avail_idx); 1417 return -EINVAL; 1418 } 1419 /* On success, callers read a descriptor at vq->last_avail_idx. 1420 * Make sure descriptor read does not bypass avail index read. */ 1421 if (num_heads) { 1422 smp_rmb(); 1423 } 1424 1425 return num_heads; 1426 } 1427 1428 /* Called within rcu_read_lock(). */ 1429 static bool virtqueue_get_head(VirtQueue *vq, unsigned int idx, 1430 unsigned int *head) 1431 { 1432 /* Grab the next descriptor number they're advertising, and increment 1433 * the index we've seen. */ 1434 *head = vring_avail_ring(vq, idx % vq->vring.num); 1435 1436 /* If their number is silly, that's a fatal mistake. */ 1437 if (*head >= vq->vring.num) { 1438 virtio_error(vq->vdev, "Guest says index %u is available", *head); 1439 return false; 1440 } 1441 1442 return true; 1443 } 1444 1445 enum { 1446 VIRTQUEUE_READ_DESC_ERROR = -1, 1447 VIRTQUEUE_READ_DESC_DONE = 0, /* end of chain */ 1448 VIRTQUEUE_READ_DESC_MORE = 1, /* more buffers in chain */ 1449 }; 1450 1451 static int virtqueue_split_read_next_desc(VirtIODevice *vdev, VRingDesc *desc, 1452 MemoryRegionCache *desc_cache, 1453 unsigned int max, unsigned int *next) 1454 { 1455 /* If this descriptor says it doesn't chain, we're done. */ 1456 if (!(desc->flags & VRING_DESC_F_NEXT)) { 1457 return VIRTQUEUE_READ_DESC_DONE; 1458 } 1459 1460 /* Check they're not leading us off end of descriptors. */ 1461 *next = desc->next; 1462 /* Make sure compiler knows to grab that: we don't want it changing! */ 1463 smp_wmb(); 1464 1465 if (*next >= max) { 1466 virtio_error(vdev, "Desc next is %u", *next); 1467 return VIRTQUEUE_READ_DESC_ERROR; 1468 } 1469 1470 vring_split_desc_read(vdev, desc, desc_cache, *next); 1471 return VIRTQUEUE_READ_DESC_MORE; 1472 } 1473 1474 /* Called within rcu_read_lock(). */ 1475 static void virtqueue_split_get_avail_bytes(VirtQueue *vq, 1476 unsigned int *in_bytes, unsigned int *out_bytes, 1477 unsigned max_in_bytes, unsigned max_out_bytes, 1478 VRingMemoryRegionCaches *caches) 1479 { 1480 VirtIODevice *vdev = vq->vdev; 1481 unsigned int max, idx; 1482 unsigned int total_bufs, in_total, out_total; 1483 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 1484 int64_t len = 0; 1485 int rc; 1486 1487 idx = vq->last_avail_idx; 1488 total_bufs = in_total = out_total = 0; 1489 1490 max = vq->vring.num; 1491 1492 while ((rc = virtqueue_num_heads(vq, idx)) > 0) { 1493 MemoryRegionCache *desc_cache = &caches->desc; 1494 unsigned int num_bufs; 1495 VRingDesc desc; 1496 unsigned int i; 1497 1498 num_bufs = total_bufs; 1499 1500 if (!virtqueue_get_head(vq, idx++, &i)) { 1501 goto err; 1502 } 1503 1504 vring_split_desc_read(vdev, &desc, desc_cache, i); 1505 1506 if (desc.flags & VRING_DESC_F_INDIRECT) { 1507 if (!desc.len || (desc.len % sizeof(VRingDesc))) { 1508 virtio_error(vdev, "Invalid size for indirect buffer table"); 1509 goto err; 1510 } 1511 1512 /* If we've got too many, that implies a descriptor loop. */ 1513 if (num_bufs >= max) { 1514 virtio_error(vdev, "Looped descriptor"); 1515 goto err; 1516 } 1517 1518 /* loop over the indirect descriptor table */ 1519 len = address_space_cache_init(&indirect_desc_cache, 1520 vdev->dma_as, 1521 desc.addr, desc.len, false); 1522 desc_cache = &indirect_desc_cache; 1523 if (len < desc.len) { 1524 virtio_error(vdev, "Cannot map indirect buffer"); 1525 goto err; 1526 } 1527 1528 max = desc.len / sizeof(VRingDesc); 1529 num_bufs = i = 0; 1530 vring_split_desc_read(vdev, &desc, desc_cache, i); 1531 } 1532 1533 do { 1534 /* If we've got too many, that implies a descriptor loop. */ 1535 if (++num_bufs > max) { 1536 virtio_error(vdev, "Looped descriptor"); 1537 goto err; 1538 } 1539 1540 if (desc.flags & VRING_DESC_F_WRITE) { 1541 in_total += desc.len; 1542 } else { 1543 out_total += desc.len; 1544 } 1545 if (in_total >= max_in_bytes && out_total >= max_out_bytes) { 1546 goto done; 1547 } 1548 1549 rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max, &i); 1550 } while (rc == VIRTQUEUE_READ_DESC_MORE); 1551 1552 if (rc == VIRTQUEUE_READ_DESC_ERROR) { 1553 goto err; 1554 } 1555 1556 if (desc_cache == &indirect_desc_cache) { 1557 address_space_cache_destroy(&indirect_desc_cache); 1558 total_bufs++; 1559 } else { 1560 total_bufs = num_bufs; 1561 } 1562 } 1563 1564 if (rc < 0) { 1565 goto err; 1566 } 1567 1568 done: 1569 address_space_cache_destroy(&indirect_desc_cache); 1570 if (in_bytes) { 1571 *in_bytes = in_total; 1572 } 1573 if (out_bytes) { 1574 *out_bytes = out_total; 1575 } 1576 return; 1577 1578 err: 1579 in_total = out_total = 0; 1580 goto done; 1581 } 1582 1583 static int virtqueue_packed_read_next_desc(VirtQueue *vq, 1584 VRingPackedDesc *desc, 1585 MemoryRegionCache 1586 *desc_cache, 1587 unsigned int max, 1588 unsigned int *next, 1589 bool indirect) 1590 { 1591 /* If this descriptor says it doesn't chain, we're done. */ 1592 if (!indirect && !(desc->flags & VRING_DESC_F_NEXT)) { 1593 return VIRTQUEUE_READ_DESC_DONE; 1594 } 1595 1596 ++*next; 1597 if (*next == max) { 1598 if (indirect) { 1599 return VIRTQUEUE_READ_DESC_DONE; 1600 } else { 1601 (*next) -= vq->vring.num; 1602 } 1603 } 1604 1605 vring_packed_desc_read(vq->vdev, desc, desc_cache, *next, false); 1606 return VIRTQUEUE_READ_DESC_MORE; 1607 } 1608 1609 /* Called within rcu_read_lock(). */ 1610 static void virtqueue_packed_get_avail_bytes(VirtQueue *vq, 1611 unsigned int *in_bytes, 1612 unsigned int *out_bytes, 1613 unsigned max_in_bytes, 1614 unsigned max_out_bytes, 1615 VRingMemoryRegionCaches *caches) 1616 { 1617 VirtIODevice *vdev = vq->vdev; 1618 unsigned int max, idx; 1619 unsigned int total_bufs, in_total, out_total; 1620 MemoryRegionCache *desc_cache; 1621 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 1622 int64_t len = 0; 1623 VRingPackedDesc desc; 1624 bool wrap_counter; 1625 1626 idx = vq->last_avail_idx; 1627 wrap_counter = vq->last_avail_wrap_counter; 1628 total_bufs = in_total = out_total = 0; 1629 1630 max = vq->vring.num; 1631 1632 for (;;) { 1633 unsigned int num_bufs = total_bufs; 1634 unsigned int i = idx; 1635 int rc; 1636 1637 desc_cache = &caches->desc; 1638 vring_packed_desc_read(vdev, &desc, desc_cache, idx, true); 1639 if (!is_desc_avail(desc.flags, wrap_counter)) { 1640 break; 1641 } 1642 1643 if (desc.flags & VRING_DESC_F_INDIRECT) { 1644 if (desc.len % sizeof(VRingPackedDesc)) { 1645 virtio_error(vdev, "Invalid size for indirect buffer table"); 1646 goto err; 1647 } 1648 1649 /* If we've got too many, that implies a descriptor loop. */ 1650 if (num_bufs >= max) { 1651 virtio_error(vdev, "Looped descriptor"); 1652 goto err; 1653 } 1654 1655 /* loop over the indirect descriptor table */ 1656 len = address_space_cache_init(&indirect_desc_cache, 1657 vdev->dma_as, 1658 desc.addr, desc.len, false); 1659 desc_cache = &indirect_desc_cache; 1660 if (len < desc.len) { 1661 virtio_error(vdev, "Cannot map indirect buffer"); 1662 goto err; 1663 } 1664 1665 max = desc.len / sizeof(VRingPackedDesc); 1666 num_bufs = i = 0; 1667 vring_packed_desc_read(vdev, &desc, desc_cache, i, false); 1668 } 1669 1670 do { 1671 /* If we've got too many, that implies a descriptor loop. */ 1672 if (++num_bufs > max) { 1673 virtio_error(vdev, "Looped descriptor"); 1674 goto err; 1675 } 1676 1677 if (desc.flags & VRING_DESC_F_WRITE) { 1678 in_total += desc.len; 1679 } else { 1680 out_total += desc.len; 1681 } 1682 if (in_total >= max_in_bytes && out_total >= max_out_bytes) { 1683 goto done; 1684 } 1685 1686 rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max, 1687 &i, desc_cache == 1688 &indirect_desc_cache); 1689 } while (rc == VIRTQUEUE_READ_DESC_MORE); 1690 1691 if (desc_cache == &indirect_desc_cache) { 1692 address_space_cache_destroy(&indirect_desc_cache); 1693 total_bufs++; 1694 idx++; 1695 } else { 1696 idx += num_bufs - total_bufs; 1697 total_bufs = num_bufs; 1698 } 1699 1700 if (idx >= vq->vring.num) { 1701 idx -= vq->vring.num; 1702 wrap_counter ^= 1; 1703 } 1704 } 1705 1706 /* Record the index and wrap counter for a kick we want */ 1707 vq->shadow_avail_idx = idx; 1708 vq->shadow_avail_wrap_counter = wrap_counter; 1709 done: 1710 address_space_cache_destroy(&indirect_desc_cache); 1711 if (in_bytes) { 1712 *in_bytes = in_total; 1713 } 1714 if (out_bytes) { 1715 *out_bytes = out_total; 1716 } 1717 return; 1718 1719 err: 1720 in_total = out_total = 0; 1721 goto done; 1722 } 1723 1724 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes, 1725 unsigned int *out_bytes, 1726 unsigned max_in_bytes, unsigned max_out_bytes) 1727 { 1728 uint16_t desc_size; 1729 VRingMemoryRegionCaches *caches; 1730 1731 RCU_READ_LOCK_GUARD(); 1732 1733 if (unlikely(!vq->vring.desc)) { 1734 goto err; 1735 } 1736 1737 caches = vring_get_region_caches(vq); 1738 if (!caches) { 1739 goto err; 1740 } 1741 1742 desc_size = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ? 1743 sizeof(VRingPackedDesc) : sizeof(VRingDesc); 1744 if (caches->desc.len < vq->vring.num * desc_size) { 1745 virtio_error(vq->vdev, "Cannot map descriptor ring"); 1746 goto err; 1747 } 1748 1749 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 1750 virtqueue_packed_get_avail_bytes(vq, in_bytes, out_bytes, 1751 max_in_bytes, max_out_bytes, 1752 caches); 1753 } else { 1754 virtqueue_split_get_avail_bytes(vq, in_bytes, out_bytes, 1755 max_in_bytes, max_out_bytes, 1756 caches); 1757 } 1758 1759 return; 1760 err: 1761 if (in_bytes) { 1762 *in_bytes = 0; 1763 } 1764 if (out_bytes) { 1765 *out_bytes = 0; 1766 } 1767 } 1768 1769 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes, 1770 unsigned int out_bytes) 1771 { 1772 unsigned int in_total, out_total; 1773 1774 virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes); 1775 return in_bytes <= in_total && out_bytes <= out_total; 1776 } 1777 1778 static bool virtqueue_map_desc(VirtIODevice *vdev, unsigned int *p_num_sg, 1779 hwaddr *addr, struct iovec *iov, 1780 unsigned int max_num_sg, bool is_write, 1781 hwaddr pa, size_t sz) 1782 { 1783 bool ok = false; 1784 unsigned num_sg = *p_num_sg; 1785 assert(num_sg <= max_num_sg); 1786 1787 if (!sz) { 1788 virtio_error(vdev, "virtio: zero sized buffers are not allowed"); 1789 goto out; 1790 } 1791 1792 while (sz) { 1793 hwaddr len = sz; 1794 1795 if (num_sg == max_num_sg) { 1796 virtio_error(vdev, "virtio: too many write descriptors in " 1797 "indirect table"); 1798 goto out; 1799 } 1800 1801 iov[num_sg].iov_base = dma_memory_map(vdev->dma_as, pa, &len, 1802 is_write ? 1803 DMA_DIRECTION_FROM_DEVICE : 1804 DMA_DIRECTION_TO_DEVICE, 1805 MEMTXATTRS_UNSPECIFIED); 1806 if (!iov[num_sg].iov_base) { 1807 virtio_error(vdev, "virtio: bogus descriptor or out of resources"); 1808 goto out; 1809 } 1810 1811 iov[num_sg].iov_len = len; 1812 addr[num_sg] = pa; 1813 1814 sz -= len; 1815 pa += len; 1816 num_sg++; 1817 } 1818 ok = true; 1819 1820 out: 1821 *p_num_sg = num_sg; 1822 return ok; 1823 } 1824 1825 /* Only used by error code paths before we have a VirtQueueElement (therefore 1826 * virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to 1827 * yet. 1828 */ 1829 static void virtqueue_undo_map_desc(unsigned int out_num, unsigned int in_num, 1830 struct iovec *iov) 1831 { 1832 unsigned int i; 1833 1834 for (i = 0; i < out_num + in_num; i++) { 1835 int is_write = i >= out_num; 1836 1837 cpu_physical_memory_unmap(iov->iov_base, iov->iov_len, is_write, 0); 1838 iov++; 1839 } 1840 } 1841 1842 static void virtqueue_map_iovec(VirtIODevice *vdev, struct iovec *sg, 1843 hwaddr *addr, unsigned int num_sg, 1844 bool is_write) 1845 { 1846 unsigned int i; 1847 hwaddr len; 1848 1849 for (i = 0; i < num_sg; i++) { 1850 len = sg[i].iov_len; 1851 sg[i].iov_base = dma_memory_map(vdev->dma_as, 1852 addr[i], &len, is_write ? 1853 DMA_DIRECTION_FROM_DEVICE : 1854 DMA_DIRECTION_TO_DEVICE, 1855 MEMTXATTRS_UNSPECIFIED); 1856 if (!sg[i].iov_base) { 1857 error_report("virtio: error trying to map MMIO memory"); 1858 exit(1); 1859 } 1860 if (len != sg[i].iov_len) { 1861 error_report("virtio: unexpected memory split"); 1862 exit(1); 1863 } 1864 } 1865 } 1866 1867 void virtqueue_map(VirtIODevice *vdev, VirtQueueElement *elem) 1868 { 1869 virtqueue_map_iovec(vdev, elem->in_sg, elem->in_addr, elem->in_num, true); 1870 virtqueue_map_iovec(vdev, elem->out_sg, elem->out_addr, elem->out_num, 1871 false); 1872 } 1873 1874 static void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num) 1875 { 1876 VirtQueueElement *elem; 1877 size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0])); 1878 size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]); 1879 size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]); 1880 size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0])); 1881 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]); 1882 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]); 1883 1884 assert(sz >= sizeof(VirtQueueElement)); 1885 elem = g_malloc(out_sg_end); 1886 trace_virtqueue_alloc_element(elem, sz, in_num, out_num); 1887 elem->out_num = out_num; 1888 elem->in_num = in_num; 1889 elem->in_addr = (void *)elem + in_addr_ofs; 1890 elem->out_addr = (void *)elem + out_addr_ofs; 1891 elem->in_sg = (void *)elem + in_sg_ofs; 1892 elem->out_sg = (void *)elem + out_sg_ofs; 1893 return elem; 1894 } 1895 1896 static void *virtqueue_split_pop(VirtQueue *vq, size_t sz) 1897 { 1898 unsigned int i, head, max; 1899 VRingMemoryRegionCaches *caches; 1900 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 1901 MemoryRegionCache *desc_cache; 1902 int64_t len; 1903 VirtIODevice *vdev = vq->vdev; 1904 VirtQueueElement *elem = NULL; 1905 unsigned out_num, in_num, elem_entries; 1906 hwaddr addr[VIRTQUEUE_MAX_SIZE]; 1907 struct iovec iov[VIRTQUEUE_MAX_SIZE]; 1908 VRingDesc desc; 1909 int rc; 1910 1911 RCU_READ_LOCK_GUARD(); 1912 if (virtio_queue_empty_rcu(vq)) { 1913 goto done; 1914 } 1915 /* Needed after virtio_queue_empty(), see comment in 1916 * virtqueue_num_heads(). */ 1917 smp_rmb(); 1918 1919 /* When we start there are none of either input nor output. */ 1920 out_num = in_num = elem_entries = 0; 1921 1922 max = vq->vring.num; 1923 1924 if (vq->inuse >= vq->vring.num) { 1925 virtio_error(vdev, "Virtqueue size exceeded"); 1926 goto done; 1927 } 1928 1929 if (!virtqueue_get_head(vq, vq->last_avail_idx++, &head)) { 1930 goto done; 1931 } 1932 1933 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 1934 vring_set_avail_event(vq, vq->last_avail_idx); 1935 } 1936 1937 i = head; 1938 1939 caches = vring_get_region_caches(vq); 1940 if (!caches) { 1941 virtio_error(vdev, "Region caches not initialized"); 1942 goto done; 1943 } 1944 1945 if (caches->desc.len < max * sizeof(VRingDesc)) { 1946 virtio_error(vdev, "Cannot map descriptor ring"); 1947 goto done; 1948 } 1949 1950 desc_cache = &caches->desc; 1951 vring_split_desc_read(vdev, &desc, desc_cache, i); 1952 if (desc.flags & VRING_DESC_F_INDIRECT) { 1953 if (!desc.len || (desc.len % sizeof(VRingDesc))) { 1954 virtio_error(vdev, "Invalid size for indirect buffer table"); 1955 goto done; 1956 } 1957 1958 /* loop over the indirect descriptor table */ 1959 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, 1960 desc.addr, desc.len, false); 1961 desc_cache = &indirect_desc_cache; 1962 if (len < desc.len) { 1963 virtio_error(vdev, "Cannot map indirect buffer"); 1964 goto done; 1965 } 1966 1967 max = desc.len / sizeof(VRingDesc); 1968 i = 0; 1969 vring_split_desc_read(vdev, &desc, desc_cache, i); 1970 } 1971 1972 /* Collect all the descriptors */ 1973 do { 1974 bool map_ok; 1975 1976 if (desc.flags & VRING_DESC_F_WRITE) { 1977 map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num, 1978 iov + out_num, 1979 VIRTQUEUE_MAX_SIZE - out_num, true, 1980 desc.addr, desc.len); 1981 } else { 1982 if (in_num) { 1983 virtio_error(vdev, "Incorrect order for descriptors"); 1984 goto err_undo_map; 1985 } 1986 map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov, 1987 VIRTQUEUE_MAX_SIZE, false, 1988 desc.addr, desc.len); 1989 } 1990 if (!map_ok) { 1991 goto err_undo_map; 1992 } 1993 1994 /* If we've got too many, that implies a descriptor loop. */ 1995 if (++elem_entries > max) { 1996 virtio_error(vdev, "Looped descriptor"); 1997 goto err_undo_map; 1998 } 1999 2000 rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max, &i); 2001 } while (rc == VIRTQUEUE_READ_DESC_MORE); 2002 2003 if (rc == VIRTQUEUE_READ_DESC_ERROR) { 2004 goto err_undo_map; 2005 } 2006 2007 /* Now copy what we have collected and mapped */ 2008 elem = virtqueue_alloc_element(sz, out_num, in_num); 2009 elem->index = head; 2010 elem->ndescs = 1; 2011 for (i = 0; i < out_num; i++) { 2012 elem->out_addr[i] = addr[i]; 2013 elem->out_sg[i] = iov[i]; 2014 } 2015 for (i = 0; i < in_num; i++) { 2016 elem->in_addr[i] = addr[out_num + i]; 2017 elem->in_sg[i] = iov[out_num + i]; 2018 } 2019 2020 vq->inuse++; 2021 2022 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); 2023 done: 2024 address_space_cache_destroy(&indirect_desc_cache); 2025 2026 return elem; 2027 2028 err_undo_map: 2029 virtqueue_undo_map_desc(out_num, in_num, iov); 2030 goto done; 2031 } 2032 2033 static void *virtqueue_packed_pop(VirtQueue *vq, size_t sz) 2034 { 2035 unsigned int i, max; 2036 VRingMemoryRegionCaches *caches; 2037 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 2038 MemoryRegionCache *desc_cache; 2039 int64_t len; 2040 VirtIODevice *vdev = vq->vdev; 2041 VirtQueueElement *elem = NULL; 2042 unsigned out_num, in_num, elem_entries; 2043 hwaddr addr[VIRTQUEUE_MAX_SIZE]; 2044 struct iovec iov[VIRTQUEUE_MAX_SIZE]; 2045 VRingPackedDesc desc; 2046 uint16_t id; 2047 int rc; 2048 2049 RCU_READ_LOCK_GUARD(); 2050 if (virtio_queue_packed_empty_rcu(vq)) { 2051 goto done; 2052 } 2053 2054 /* When we start there are none of either input nor output. */ 2055 out_num = in_num = elem_entries = 0; 2056 2057 max = vq->vring.num; 2058 2059 if (vq->inuse >= vq->vring.num) { 2060 virtio_error(vdev, "Virtqueue size exceeded"); 2061 goto done; 2062 } 2063 2064 i = vq->last_avail_idx; 2065 2066 caches = vring_get_region_caches(vq); 2067 if (!caches) { 2068 virtio_error(vdev, "Region caches not initialized"); 2069 goto done; 2070 } 2071 2072 if (caches->desc.len < max * sizeof(VRingDesc)) { 2073 virtio_error(vdev, "Cannot map descriptor ring"); 2074 goto done; 2075 } 2076 2077 desc_cache = &caches->desc; 2078 vring_packed_desc_read(vdev, &desc, desc_cache, i, true); 2079 id = desc.id; 2080 if (desc.flags & VRING_DESC_F_INDIRECT) { 2081 if (desc.len % sizeof(VRingPackedDesc)) { 2082 virtio_error(vdev, "Invalid size for indirect buffer table"); 2083 goto done; 2084 } 2085 2086 /* loop over the indirect descriptor table */ 2087 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, 2088 desc.addr, desc.len, false); 2089 desc_cache = &indirect_desc_cache; 2090 if (len < desc.len) { 2091 virtio_error(vdev, "Cannot map indirect buffer"); 2092 goto done; 2093 } 2094 2095 max = desc.len / sizeof(VRingPackedDesc); 2096 i = 0; 2097 vring_packed_desc_read(vdev, &desc, desc_cache, i, false); 2098 } 2099 2100 /* Collect all the descriptors */ 2101 do { 2102 bool map_ok; 2103 2104 if (desc.flags & VRING_DESC_F_WRITE) { 2105 map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num, 2106 iov + out_num, 2107 VIRTQUEUE_MAX_SIZE - out_num, true, 2108 desc.addr, desc.len); 2109 } else { 2110 if (in_num) { 2111 virtio_error(vdev, "Incorrect order for descriptors"); 2112 goto err_undo_map; 2113 } 2114 map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov, 2115 VIRTQUEUE_MAX_SIZE, false, 2116 desc.addr, desc.len); 2117 } 2118 if (!map_ok) { 2119 goto err_undo_map; 2120 } 2121 2122 /* If we've got too many, that implies a descriptor loop. */ 2123 if (++elem_entries > max) { 2124 virtio_error(vdev, "Looped descriptor"); 2125 goto err_undo_map; 2126 } 2127 2128 rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max, &i, 2129 desc_cache == 2130 &indirect_desc_cache); 2131 } while (rc == VIRTQUEUE_READ_DESC_MORE); 2132 2133 /* Now copy what we have collected and mapped */ 2134 elem = virtqueue_alloc_element(sz, out_num, in_num); 2135 for (i = 0; i < out_num; i++) { 2136 elem->out_addr[i] = addr[i]; 2137 elem->out_sg[i] = iov[i]; 2138 } 2139 for (i = 0; i < in_num; i++) { 2140 elem->in_addr[i] = addr[out_num + i]; 2141 elem->in_sg[i] = iov[out_num + i]; 2142 } 2143 2144 elem->index = id; 2145 elem->ndescs = (desc_cache == &indirect_desc_cache) ? 1 : elem_entries; 2146 vq->last_avail_idx += elem->ndescs; 2147 vq->inuse += elem->ndescs; 2148 2149 if (vq->last_avail_idx >= vq->vring.num) { 2150 vq->last_avail_idx -= vq->vring.num; 2151 vq->last_avail_wrap_counter ^= 1; 2152 } 2153 2154 vq->shadow_avail_idx = vq->last_avail_idx; 2155 vq->shadow_avail_wrap_counter = vq->last_avail_wrap_counter; 2156 2157 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); 2158 done: 2159 address_space_cache_destroy(&indirect_desc_cache); 2160 2161 return elem; 2162 2163 err_undo_map: 2164 virtqueue_undo_map_desc(out_num, in_num, iov); 2165 goto done; 2166 } 2167 2168 void *virtqueue_pop(VirtQueue *vq, size_t sz) 2169 { 2170 if (virtio_device_disabled(vq->vdev)) { 2171 return NULL; 2172 } 2173 2174 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { 2175 return virtqueue_packed_pop(vq, sz); 2176 } else { 2177 return virtqueue_split_pop(vq, sz); 2178 } 2179 } 2180 2181 static unsigned int virtqueue_packed_drop_all(VirtQueue *vq) 2182 { 2183 VRingMemoryRegionCaches *caches; 2184 MemoryRegionCache *desc_cache; 2185 unsigned int dropped = 0; 2186 VirtQueueElement elem = {}; 2187 VirtIODevice *vdev = vq->vdev; 2188 VRingPackedDesc desc; 2189 2190 RCU_READ_LOCK_GUARD(); 2191 2192 caches = vring_get_region_caches(vq); 2193 if (!caches) { 2194 return 0; 2195 } 2196 2197 desc_cache = &caches->desc; 2198 2199 virtio_queue_set_notification(vq, 0); 2200 2201 while (vq->inuse < vq->vring.num) { 2202 unsigned int idx = vq->last_avail_idx; 2203 /* 2204 * works similar to virtqueue_pop but does not map buffers 2205 * and does not allocate any memory. 2206 */ 2207 vring_packed_desc_read(vdev, &desc, desc_cache, 2208 vq->last_avail_idx , true); 2209 if (!is_desc_avail(desc.flags, vq->last_avail_wrap_counter)) { 2210 break; 2211 } 2212 elem.index = desc.id; 2213 elem.ndescs = 1; 2214 while (virtqueue_packed_read_next_desc(vq, &desc, desc_cache, 2215 vq->vring.num, &idx, false)) { 2216 ++elem.ndescs; 2217 } 2218 /* 2219 * immediately push the element, nothing to unmap 2220 * as both in_num and out_num are set to 0. 2221 */ 2222 virtqueue_push(vq, &elem, 0); 2223 dropped++; 2224 vq->last_avail_idx += elem.ndescs; 2225 if (vq->last_avail_idx >= vq->vring.num) { 2226 vq->last_avail_idx -= vq->vring.num; 2227 vq->last_avail_wrap_counter ^= 1; 2228 } 2229 } 2230 2231 return dropped; 2232 } 2233 2234 static unsigned int virtqueue_split_drop_all(VirtQueue *vq) 2235 { 2236 unsigned int dropped = 0; 2237 VirtQueueElement elem = {}; 2238 VirtIODevice *vdev = vq->vdev; 2239 bool fEventIdx = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); 2240 2241 while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) { 2242 /* works similar to virtqueue_pop but does not map buffers 2243 * and does not allocate any memory */ 2244 smp_rmb(); 2245 if (!virtqueue_get_head(vq, vq->last_avail_idx, &elem.index)) { 2246 break; 2247 } 2248 vq->inuse++; 2249 vq->last_avail_idx++; 2250 if (fEventIdx) { 2251 vring_set_avail_event(vq, vq->last_avail_idx); 2252 } 2253 /* immediately push the element, nothing to unmap 2254 * as both in_num and out_num are set to 0 */ 2255 virtqueue_push(vq, &elem, 0); 2256 dropped++; 2257 } 2258 2259 return dropped; 2260 } 2261 2262 /* virtqueue_drop_all: 2263 * @vq: The #VirtQueue 2264 * Drops all queued buffers and indicates them to the guest 2265 * as if they are done. Useful when buffers can not be 2266 * processed but must be returned to the guest. 2267 */ 2268 unsigned int virtqueue_drop_all(VirtQueue *vq) 2269 { 2270 struct VirtIODevice *vdev = vq->vdev; 2271 2272 if (virtio_device_disabled(vq->vdev)) { 2273 return 0; 2274 } 2275 2276 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 2277 return virtqueue_packed_drop_all(vq); 2278 } else { 2279 return virtqueue_split_drop_all(vq); 2280 } 2281 } 2282 2283 /* Reading and writing a structure directly to QEMUFile is *awful*, but 2284 * it is what QEMU has always done by mistake. We can change it sooner 2285 * or later by bumping the version number of the affected vm states. 2286 * In the meanwhile, since the in-memory layout of VirtQueueElement 2287 * has changed, we need to marshal to and from the layout that was 2288 * used before the change. 2289 */ 2290 typedef struct VirtQueueElementOld { 2291 unsigned int index; 2292 unsigned int out_num; 2293 unsigned int in_num; 2294 hwaddr in_addr[VIRTQUEUE_MAX_SIZE]; 2295 hwaddr out_addr[VIRTQUEUE_MAX_SIZE]; 2296 struct iovec in_sg[VIRTQUEUE_MAX_SIZE]; 2297 struct iovec out_sg[VIRTQUEUE_MAX_SIZE]; 2298 } VirtQueueElementOld; 2299 2300 void *qemu_get_virtqueue_element(VirtIODevice *vdev, QEMUFile *f, size_t sz) 2301 { 2302 VirtQueueElement *elem; 2303 VirtQueueElementOld data; 2304 int i; 2305 2306 qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld)); 2307 2308 /* TODO: teach all callers that this can fail, and return failure instead 2309 * of asserting here. 2310 * This is just one thing (there are probably more) that must be 2311 * fixed before we can allow NDEBUG compilation. 2312 */ 2313 assert(ARRAY_SIZE(data.in_addr) >= data.in_num); 2314 assert(ARRAY_SIZE(data.out_addr) >= data.out_num); 2315 2316 elem = virtqueue_alloc_element(sz, data.out_num, data.in_num); 2317 elem->index = data.index; 2318 2319 for (i = 0; i < elem->in_num; i++) { 2320 elem->in_addr[i] = data.in_addr[i]; 2321 } 2322 2323 for (i = 0; i < elem->out_num; i++) { 2324 elem->out_addr[i] = data.out_addr[i]; 2325 } 2326 2327 for (i = 0; i < elem->in_num; i++) { 2328 /* Base is overwritten by virtqueue_map. */ 2329 elem->in_sg[i].iov_base = 0; 2330 elem->in_sg[i].iov_len = data.in_sg[i].iov_len; 2331 } 2332 2333 for (i = 0; i < elem->out_num; i++) { 2334 /* Base is overwritten by virtqueue_map. */ 2335 elem->out_sg[i].iov_base = 0; 2336 elem->out_sg[i].iov_len = data.out_sg[i].iov_len; 2337 } 2338 2339 if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 2340 qemu_get_be32s(f, &elem->ndescs); 2341 } 2342 2343 virtqueue_map(vdev, elem); 2344 return elem; 2345 } 2346 2347 void qemu_put_virtqueue_element(VirtIODevice *vdev, QEMUFile *f, 2348 VirtQueueElement *elem) 2349 { 2350 VirtQueueElementOld data; 2351 int i; 2352 2353 memset(&data, 0, sizeof(data)); 2354 data.index = elem->index; 2355 data.in_num = elem->in_num; 2356 data.out_num = elem->out_num; 2357 2358 for (i = 0; i < elem->in_num; i++) { 2359 data.in_addr[i] = elem->in_addr[i]; 2360 } 2361 2362 for (i = 0; i < elem->out_num; i++) { 2363 data.out_addr[i] = elem->out_addr[i]; 2364 } 2365 2366 for (i = 0; i < elem->in_num; i++) { 2367 /* Base is overwritten by virtqueue_map when loading. Do not 2368 * save it, as it would leak the QEMU address space layout. */ 2369 data.in_sg[i].iov_len = elem->in_sg[i].iov_len; 2370 } 2371 2372 for (i = 0; i < elem->out_num; i++) { 2373 /* Do not save iov_base as above. */ 2374 data.out_sg[i].iov_len = elem->out_sg[i].iov_len; 2375 } 2376 2377 if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 2378 qemu_put_be32s(f, &elem->ndescs); 2379 } 2380 2381 qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld)); 2382 } 2383 2384 /* virtio device */ 2385 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector) 2386 { 2387 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2388 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 2389 2390 if (virtio_device_disabled(vdev)) { 2391 return; 2392 } 2393 2394 if (k->notify) { 2395 k->notify(qbus->parent, vector); 2396 } 2397 } 2398 2399 void virtio_update_irq(VirtIODevice *vdev) 2400 { 2401 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); 2402 } 2403 2404 static int virtio_validate_features(VirtIODevice *vdev) 2405 { 2406 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2407 2408 if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM) && 2409 !virtio_vdev_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) { 2410 return -EFAULT; 2411 } 2412 2413 if (k->validate_features) { 2414 return k->validate_features(vdev); 2415 } else { 2416 return 0; 2417 } 2418 } 2419 2420 int virtio_set_status(VirtIODevice *vdev, uint8_t val) 2421 { 2422 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2423 trace_virtio_set_status(vdev, val); 2424 2425 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 2426 if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) && 2427 val & VIRTIO_CONFIG_S_FEATURES_OK) { 2428 int ret = virtio_validate_features(vdev); 2429 2430 if (ret) { 2431 return ret; 2432 } 2433 } 2434 } 2435 2436 if ((vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) != 2437 (val & VIRTIO_CONFIG_S_DRIVER_OK)) { 2438 virtio_set_started(vdev, val & VIRTIO_CONFIG_S_DRIVER_OK); 2439 } 2440 2441 if (k->set_status) { 2442 k->set_status(vdev, val); 2443 } 2444 vdev->status = val; 2445 2446 return 0; 2447 } 2448 2449 static enum virtio_device_endian virtio_default_endian(void) 2450 { 2451 if (target_words_bigendian()) { 2452 return VIRTIO_DEVICE_ENDIAN_BIG; 2453 } else { 2454 return VIRTIO_DEVICE_ENDIAN_LITTLE; 2455 } 2456 } 2457 2458 static enum virtio_device_endian virtio_current_cpu_endian(void) 2459 { 2460 if (cpu_virtio_is_big_endian(current_cpu)) { 2461 return VIRTIO_DEVICE_ENDIAN_BIG; 2462 } else { 2463 return VIRTIO_DEVICE_ENDIAN_LITTLE; 2464 } 2465 } 2466 2467 static void __virtio_queue_reset(VirtIODevice *vdev, uint32_t i) 2468 { 2469 vdev->vq[i].vring.desc = 0; 2470 vdev->vq[i].vring.avail = 0; 2471 vdev->vq[i].vring.used = 0; 2472 vdev->vq[i].last_avail_idx = 0; 2473 vdev->vq[i].shadow_avail_idx = 0; 2474 vdev->vq[i].used_idx = 0; 2475 vdev->vq[i].last_avail_wrap_counter = true; 2476 vdev->vq[i].shadow_avail_wrap_counter = true; 2477 vdev->vq[i].used_wrap_counter = true; 2478 virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR); 2479 vdev->vq[i].signalled_used = 0; 2480 vdev->vq[i].signalled_used_valid = false; 2481 vdev->vq[i].notification = true; 2482 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default; 2483 vdev->vq[i].inuse = 0; 2484 virtio_virtqueue_reset_region_cache(&vdev->vq[i]); 2485 } 2486 2487 void virtio_reset(void *opaque) 2488 { 2489 VirtIODevice *vdev = opaque; 2490 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2491 int i; 2492 2493 virtio_set_status(vdev, 0); 2494 if (current_cpu) { 2495 /* Guest initiated reset */ 2496 vdev->device_endian = virtio_current_cpu_endian(); 2497 } else { 2498 /* System reset */ 2499 vdev->device_endian = virtio_default_endian(); 2500 } 2501 2502 if (k->reset) { 2503 k->reset(vdev); 2504 } 2505 2506 vdev->start_on_kick = false; 2507 vdev->started = false; 2508 vdev->broken = false; 2509 vdev->guest_features = 0; 2510 vdev->queue_sel = 0; 2511 vdev->status = 0; 2512 vdev->disabled = false; 2513 qatomic_set(&vdev->isr, 0); 2514 vdev->config_vector = VIRTIO_NO_VECTOR; 2515 virtio_notify_vector(vdev, vdev->config_vector); 2516 2517 for(i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2518 __virtio_queue_reset(vdev, i); 2519 } 2520 } 2521 2522 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr) 2523 { 2524 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2525 uint8_t val; 2526 2527 if (addr + sizeof(val) > vdev->config_len) { 2528 return (uint32_t)-1; 2529 } 2530 2531 k->get_config(vdev, vdev->config); 2532 2533 val = ldub_p(vdev->config + addr); 2534 return val; 2535 } 2536 2537 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr) 2538 { 2539 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2540 uint16_t val; 2541 2542 if (addr + sizeof(val) > vdev->config_len) { 2543 return (uint32_t)-1; 2544 } 2545 2546 k->get_config(vdev, vdev->config); 2547 2548 val = lduw_p(vdev->config + addr); 2549 return val; 2550 } 2551 2552 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr) 2553 { 2554 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2555 uint32_t val; 2556 2557 if (addr + sizeof(val) > vdev->config_len) { 2558 return (uint32_t)-1; 2559 } 2560 2561 k->get_config(vdev, vdev->config); 2562 2563 val = ldl_p(vdev->config + addr); 2564 return val; 2565 } 2566 2567 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data) 2568 { 2569 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2570 uint8_t val = data; 2571 2572 if (addr + sizeof(val) > vdev->config_len) { 2573 return; 2574 } 2575 2576 stb_p(vdev->config + addr, val); 2577 2578 if (k->set_config) { 2579 k->set_config(vdev, vdev->config); 2580 } 2581 } 2582 2583 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data) 2584 { 2585 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2586 uint16_t val = data; 2587 2588 if (addr + sizeof(val) > vdev->config_len) { 2589 return; 2590 } 2591 2592 stw_p(vdev->config + addr, val); 2593 2594 if (k->set_config) { 2595 k->set_config(vdev, vdev->config); 2596 } 2597 } 2598 2599 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data) 2600 { 2601 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2602 uint32_t val = data; 2603 2604 if (addr + sizeof(val) > vdev->config_len) { 2605 return; 2606 } 2607 2608 stl_p(vdev->config + addr, val); 2609 2610 if (k->set_config) { 2611 k->set_config(vdev, vdev->config); 2612 } 2613 } 2614 2615 uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr) 2616 { 2617 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2618 uint8_t val; 2619 2620 if (addr + sizeof(val) > vdev->config_len) { 2621 return (uint32_t)-1; 2622 } 2623 2624 k->get_config(vdev, vdev->config); 2625 2626 val = ldub_p(vdev->config + addr); 2627 return val; 2628 } 2629 2630 uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr) 2631 { 2632 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2633 uint16_t val; 2634 2635 if (addr + sizeof(val) > vdev->config_len) { 2636 return (uint32_t)-1; 2637 } 2638 2639 k->get_config(vdev, vdev->config); 2640 2641 val = lduw_le_p(vdev->config + addr); 2642 return val; 2643 } 2644 2645 uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr) 2646 { 2647 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2648 uint32_t val; 2649 2650 if (addr + sizeof(val) > vdev->config_len) { 2651 return (uint32_t)-1; 2652 } 2653 2654 k->get_config(vdev, vdev->config); 2655 2656 val = ldl_le_p(vdev->config + addr); 2657 return val; 2658 } 2659 2660 void virtio_config_modern_writeb(VirtIODevice *vdev, 2661 uint32_t addr, uint32_t data) 2662 { 2663 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2664 uint8_t val = data; 2665 2666 if (addr + sizeof(val) > vdev->config_len) { 2667 return; 2668 } 2669 2670 stb_p(vdev->config + addr, val); 2671 2672 if (k->set_config) { 2673 k->set_config(vdev, vdev->config); 2674 } 2675 } 2676 2677 void virtio_config_modern_writew(VirtIODevice *vdev, 2678 uint32_t addr, uint32_t data) 2679 { 2680 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2681 uint16_t val = data; 2682 2683 if (addr + sizeof(val) > vdev->config_len) { 2684 return; 2685 } 2686 2687 stw_le_p(vdev->config + addr, val); 2688 2689 if (k->set_config) { 2690 k->set_config(vdev, vdev->config); 2691 } 2692 } 2693 2694 void virtio_config_modern_writel(VirtIODevice *vdev, 2695 uint32_t addr, uint32_t data) 2696 { 2697 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2698 uint32_t val = data; 2699 2700 if (addr + sizeof(val) > vdev->config_len) { 2701 return; 2702 } 2703 2704 stl_le_p(vdev->config + addr, val); 2705 2706 if (k->set_config) { 2707 k->set_config(vdev, vdev->config); 2708 } 2709 } 2710 2711 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr) 2712 { 2713 if (!vdev->vq[n].vring.num) { 2714 return; 2715 } 2716 vdev->vq[n].vring.desc = addr; 2717 virtio_queue_update_rings(vdev, n); 2718 } 2719 2720 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n) 2721 { 2722 return vdev->vq[n].vring.desc; 2723 } 2724 2725 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc, 2726 hwaddr avail, hwaddr used) 2727 { 2728 if (!vdev->vq[n].vring.num) { 2729 return; 2730 } 2731 vdev->vq[n].vring.desc = desc; 2732 vdev->vq[n].vring.avail = avail; 2733 vdev->vq[n].vring.used = used; 2734 virtio_init_region_cache(vdev, n); 2735 } 2736 2737 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num) 2738 { 2739 /* Don't allow guest to flip queue between existent and 2740 * nonexistent states, or to set it to an invalid size. 2741 */ 2742 if (!!num != !!vdev->vq[n].vring.num || 2743 num > VIRTQUEUE_MAX_SIZE || 2744 num < 0) { 2745 return; 2746 } 2747 vdev->vq[n].vring.num = num; 2748 } 2749 2750 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector) 2751 { 2752 return QLIST_FIRST(&vdev->vector_queues[vector]); 2753 } 2754 2755 VirtQueue *virtio_vector_next_queue(VirtQueue *vq) 2756 { 2757 return QLIST_NEXT(vq, node); 2758 } 2759 2760 int virtio_queue_get_num(VirtIODevice *vdev, int n) 2761 { 2762 return vdev->vq[n].vring.num; 2763 } 2764 2765 int virtio_queue_get_max_num(VirtIODevice *vdev, int n) 2766 { 2767 return vdev->vq[n].vring.num_default; 2768 } 2769 2770 int virtio_get_num_queues(VirtIODevice *vdev) 2771 { 2772 int i; 2773 2774 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2775 if (!virtio_queue_get_num(vdev, i)) { 2776 break; 2777 } 2778 } 2779 2780 return i; 2781 } 2782 2783 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align) 2784 { 2785 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2786 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 2787 2788 /* virtio-1 compliant devices cannot change the alignment */ 2789 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 2790 error_report("tried to modify queue alignment for virtio-1 device"); 2791 return; 2792 } 2793 /* Check that the transport told us it was going to do this 2794 * (so a buggy transport will immediately assert rather than 2795 * silently failing to migrate this state) 2796 */ 2797 assert(k->has_variable_vring_alignment); 2798 2799 if (align) { 2800 vdev->vq[n].vring.align = align; 2801 virtio_queue_update_rings(vdev, n); 2802 } 2803 } 2804 2805 static void virtio_queue_notify_vq(VirtQueue *vq) 2806 { 2807 if (vq->vring.desc && vq->handle_output) { 2808 VirtIODevice *vdev = vq->vdev; 2809 2810 if (unlikely(vdev->broken)) { 2811 return; 2812 } 2813 2814 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); 2815 vq->handle_output(vdev, vq); 2816 2817 if (unlikely(vdev->start_on_kick)) { 2818 virtio_set_started(vdev, true); 2819 } 2820 } 2821 } 2822 2823 void virtio_queue_notify(VirtIODevice *vdev, int n) 2824 { 2825 VirtQueue *vq = &vdev->vq[n]; 2826 2827 if (unlikely(!vq->vring.desc || vdev->broken)) { 2828 return; 2829 } 2830 2831 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); 2832 if (vq->host_notifier_enabled) { 2833 event_notifier_set(&vq->host_notifier); 2834 } else if (vq->handle_output) { 2835 vq->handle_output(vdev, vq); 2836 2837 if (unlikely(vdev->start_on_kick)) { 2838 virtio_set_started(vdev, true); 2839 } 2840 } 2841 } 2842 2843 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n) 2844 { 2845 return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector : 2846 VIRTIO_NO_VECTOR; 2847 } 2848 2849 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector) 2850 { 2851 VirtQueue *vq = &vdev->vq[n]; 2852 2853 if (n < VIRTIO_QUEUE_MAX) { 2854 if (vdev->vector_queues && 2855 vdev->vq[n].vector != VIRTIO_NO_VECTOR) { 2856 QLIST_REMOVE(vq, node); 2857 } 2858 vdev->vq[n].vector = vector; 2859 if (vdev->vector_queues && 2860 vector != VIRTIO_NO_VECTOR) { 2861 QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node); 2862 } 2863 } 2864 } 2865 2866 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size, 2867 VirtIOHandleOutput handle_output) 2868 { 2869 int i; 2870 2871 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2872 if (vdev->vq[i].vring.num == 0) 2873 break; 2874 } 2875 2876 if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE) 2877 abort(); 2878 2879 vdev->vq[i].vring.num = queue_size; 2880 vdev->vq[i].vring.num_default = queue_size; 2881 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN; 2882 vdev->vq[i].handle_output = handle_output; 2883 vdev->vq[i].used_elems = g_new0(VirtQueueElement, queue_size); 2884 2885 return &vdev->vq[i]; 2886 } 2887 2888 void virtio_delete_queue(VirtQueue *vq) 2889 { 2890 vq->vring.num = 0; 2891 vq->vring.num_default = 0; 2892 vq->handle_output = NULL; 2893 g_free(vq->used_elems); 2894 vq->used_elems = NULL; 2895 virtio_virtqueue_reset_region_cache(vq); 2896 } 2897 2898 void virtio_del_queue(VirtIODevice *vdev, int n) 2899 { 2900 if (n < 0 || n >= VIRTIO_QUEUE_MAX) { 2901 abort(); 2902 } 2903 2904 virtio_delete_queue(&vdev->vq[n]); 2905 } 2906 2907 static void virtio_set_isr(VirtIODevice *vdev, int value) 2908 { 2909 uint8_t old = qatomic_read(&vdev->isr); 2910 2911 /* Do not write ISR if it does not change, so that its cacheline remains 2912 * shared in the common case where the guest does not read it. 2913 */ 2914 if ((old & value) != value) { 2915 qatomic_or(&vdev->isr, value); 2916 } 2917 } 2918 2919 /* Called within rcu_read_lock(). */ 2920 static bool virtio_split_should_notify(VirtIODevice *vdev, VirtQueue *vq) 2921 { 2922 uint16_t old, new; 2923 bool v; 2924 /* We need to expose used array entries before checking used event. */ 2925 smp_mb(); 2926 /* Always notify when queue is empty (when feature acknowledge) */ 2927 if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) && 2928 !vq->inuse && virtio_queue_empty(vq)) { 2929 return true; 2930 } 2931 2932 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 2933 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT); 2934 } 2935 2936 v = vq->signalled_used_valid; 2937 vq->signalled_used_valid = true; 2938 old = vq->signalled_used; 2939 new = vq->signalled_used = vq->used_idx; 2940 return !v || vring_need_event(vring_get_used_event(vq), new, old); 2941 } 2942 2943 static bool vring_packed_need_event(VirtQueue *vq, bool wrap, 2944 uint16_t off_wrap, uint16_t new, 2945 uint16_t old) 2946 { 2947 int off = off_wrap & ~(1 << 15); 2948 2949 if (wrap != off_wrap >> 15) { 2950 off -= vq->vring.num; 2951 } 2952 2953 return vring_need_event(off, new, old); 2954 } 2955 2956 /* Called within rcu_read_lock(). */ 2957 static bool virtio_packed_should_notify(VirtIODevice *vdev, VirtQueue *vq) 2958 { 2959 VRingPackedDescEvent e; 2960 uint16_t old, new; 2961 bool v; 2962 VRingMemoryRegionCaches *caches; 2963 2964 caches = vring_get_region_caches(vq); 2965 if (!caches) { 2966 return false; 2967 } 2968 2969 vring_packed_event_read(vdev, &caches->avail, &e); 2970 2971 old = vq->signalled_used; 2972 new = vq->signalled_used = vq->used_idx; 2973 v = vq->signalled_used_valid; 2974 vq->signalled_used_valid = true; 2975 2976 if (e.flags == VRING_PACKED_EVENT_FLAG_DISABLE) { 2977 return false; 2978 } else if (e.flags == VRING_PACKED_EVENT_FLAG_ENABLE) { 2979 return true; 2980 } 2981 2982 return !v || vring_packed_need_event(vq, vq->used_wrap_counter, 2983 e.off_wrap, new, old); 2984 } 2985 2986 /* Called within rcu_read_lock(). */ 2987 static bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq) 2988 { 2989 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 2990 return virtio_packed_should_notify(vdev, vq); 2991 } else { 2992 return virtio_split_should_notify(vdev, vq); 2993 } 2994 } 2995 2996 void virtio_notify_irqfd(VirtIODevice *vdev, VirtQueue *vq) 2997 { 2998 WITH_RCU_READ_LOCK_GUARD() { 2999 if (!virtio_should_notify(vdev, vq)) { 3000 return; 3001 } 3002 } 3003 3004 trace_virtio_notify_irqfd(vdev, vq); 3005 3006 /* 3007 * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but 3008 * windows drivers included in virtio-win 1.8.0 (circa 2015) are 3009 * incorrectly polling this bit during crashdump and hibernation 3010 * in MSI mode, causing a hang if this bit is never updated. 3011 * Recent releases of Windows do not really shut down, but rather 3012 * log out and hibernate to make the next startup faster. Hence, 3013 * this manifested as a more serious hang during shutdown with 3014 * 3015 * Next driver release from 2016 fixed this problem, so working around it 3016 * is not a must, but it's easy to do so let's do it here. 3017 * 3018 * Note: it's safe to update ISR from any thread as it was switched 3019 * to an atomic operation. 3020 */ 3021 virtio_set_isr(vq->vdev, 0x1); 3022 event_notifier_set(&vq->guest_notifier); 3023 } 3024 3025 static void virtio_irq(VirtQueue *vq) 3026 { 3027 virtio_set_isr(vq->vdev, 0x1); 3028 virtio_notify_vector(vq->vdev, vq->vector); 3029 } 3030 3031 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq) 3032 { 3033 WITH_RCU_READ_LOCK_GUARD() { 3034 if (!virtio_should_notify(vdev, vq)) { 3035 return; 3036 } 3037 } 3038 3039 trace_virtio_notify(vdev, vq); 3040 virtio_irq(vq); 3041 } 3042 3043 void virtio_notify_config(VirtIODevice *vdev) 3044 { 3045 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 3046 return; 3047 3048 virtio_set_isr(vdev, 0x3); 3049 vdev->generation++; 3050 virtio_notify_vector(vdev, vdev->config_vector); 3051 } 3052 3053 static bool virtio_device_endian_needed(void *opaque) 3054 { 3055 VirtIODevice *vdev = opaque; 3056 3057 assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN); 3058 if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 3059 return vdev->device_endian != virtio_default_endian(); 3060 } 3061 /* Devices conforming to VIRTIO 1.0 or later are always LE. */ 3062 return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE; 3063 } 3064 3065 static bool virtio_64bit_features_needed(void *opaque) 3066 { 3067 VirtIODevice *vdev = opaque; 3068 3069 return (vdev->host_features >> 32) != 0; 3070 } 3071 3072 static bool virtio_virtqueue_needed(void *opaque) 3073 { 3074 VirtIODevice *vdev = opaque; 3075 3076 return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1); 3077 } 3078 3079 static bool virtio_packed_virtqueue_needed(void *opaque) 3080 { 3081 VirtIODevice *vdev = opaque; 3082 3083 return virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED); 3084 } 3085 3086 static bool virtio_ringsize_needed(void *opaque) 3087 { 3088 VirtIODevice *vdev = opaque; 3089 int i; 3090 3091 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 3092 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) { 3093 return true; 3094 } 3095 } 3096 return false; 3097 } 3098 3099 static bool virtio_extra_state_needed(void *opaque) 3100 { 3101 VirtIODevice *vdev = opaque; 3102 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3103 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3104 3105 return k->has_extra_state && 3106 k->has_extra_state(qbus->parent); 3107 } 3108 3109 static bool virtio_broken_needed(void *opaque) 3110 { 3111 VirtIODevice *vdev = opaque; 3112 3113 return vdev->broken; 3114 } 3115 3116 static bool virtio_started_needed(void *opaque) 3117 { 3118 VirtIODevice *vdev = opaque; 3119 3120 return vdev->started; 3121 } 3122 3123 static bool virtio_disabled_needed(void *opaque) 3124 { 3125 VirtIODevice *vdev = opaque; 3126 3127 return vdev->disabled; 3128 } 3129 3130 static const VMStateDescription vmstate_virtqueue = { 3131 .name = "virtqueue_state", 3132 .version_id = 1, 3133 .minimum_version_id = 1, 3134 .fields = (VMStateField[]) { 3135 VMSTATE_UINT64(vring.avail, struct VirtQueue), 3136 VMSTATE_UINT64(vring.used, struct VirtQueue), 3137 VMSTATE_END_OF_LIST() 3138 } 3139 }; 3140 3141 static const VMStateDescription vmstate_packed_virtqueue = { 3142 .name = "packed_virtqueue_state", 3143 .version_id = 1, 3144 .minimum_version_id = 1, 3145 .fields = (VMStateField[]) { 3146 VMSTATE_UINT16(last_avail_idx, struct VirtQueue), 3147 VMSTATE_BOOL(last_avail_wrap_counter, struct VirtQueue), 3148 VMSTATE_UINT16(used_idx, struct VirtQueue), 3149 VMSTATE_BOOL(used_wrap_counter, struct VirtQueue), 3150 VMSTATE_UINT32(inuse, struct VirtQueue), 3151 VMSTATE_END_OF_LIST() 3152 } 3153 }; 3154 3155 static const VMStateDescription vmstate_virtio_virtqueues = { 3156 .name = "virtio/virtqueues", 3157 .version_id = 1, 3158 .minimum_version_id = 1, 3159 .needed = &virtio_virtqueue_needed, 3160 .fields = (VMStateField[]) { 3161 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, 3162 VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue), 3163 VMSTATE_END_OF_LIST() 3164 } 3165 }; 3166 3167 static const VMStateDescription vmstate_virtio_packed_virtqueues = { 3168 .name = "virtio/packed_virtqueues", 3169 .version_id = 1, 3170 .minimum_version_id = 1, 3171 .needed = &virtio_packed_virtqueue_needed, 3172 .fields = (VMStateField[]) { 3173 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, 3174 VIRTIO_QUEUE_MAX, 0, vmstate_packed_virtqueue, VirtQueue), 3175 VMSTATE_END_OF_LIST() 3176 } 3177 }; 3178 3179 static const VMStateDescription vmstate_ringsize = { 3180 .name = "ringsize_state", 3181 .version_id = 1, 3182 .minimum_version_id = 1, 3183 .fields = (VMStateField[]) { 3184 VMSTATE_UINT32(vring.num_default, struct VirtQueue), 3185 VMSTATE_END_OF_LIST() 3186 } 3187 }; 3188 3189 static const VMStateDescription vmstate_virtio_ringsize = { 3190 .name = "virtio/ringsize", 3191 .version_id = 1, 3192 .minimum_version_id = 1, 3193 .needed = &virtio_ringsize_needed, 3194 .fields = (VMStateField[]) { 3195 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, 3196 VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue), 3197 VMSTATE_END_OF_LIST() 3198 } 3199 }; 3200 3201 static int get_extra_state(QEMUFile *f, void *pv, size_t size, 3202 const VMStateField *field) 3203 { 3204 VirtIODevice *vdev = pv; 3205 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3206 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3207 3208 if (!k->load_extra_state) { 3209 return -1; 3210 } else { 3211 return k->load_extra_state(qbus->parent, f); 3212 } 3213 } 3214 3215 static int put_extra_state(QEMUFile *f, void *pv, size_t size, 3216 const VMStateField *field, JSONWriter *vmdesc) 3217 { 3218 VirtIODevice *vdev = pv; 3219 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3220 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3221 3222 k->save_extra_state(qbus->parent, f); 3223 return 0; 3224 } 3225 3226 static const VMStateInfo vmstate_info_extra_state = { 3227 .name = "virtqueue_extra_state", 3228 .get = get_extra_state, 3229 .put = put_extra_state, 3230 }; 3231 3232 static const VMStateDescription vmstate_virtio_extra_state = { 3233 .name = "virtio/extra_state", 3234 .version_id = 1, 3235 .minimum_version_id = 1, 3236 .needed = &virtio_extra_state_needed, 3237 .fields = (VMStateField[]) { 3238 { 3239 .name = "extra_state", 3240 .version_id = 0, 3241 .field_exists = NULL, 3242 .size = 0, 3243 .info = &vmstate_info_extra_state, 3244 .flags = VMS_SINGLE, 3245 .offset = 0, 3246 }, 3247 VMSTATE_END_OF_LIST() 3248 } 3249 }; 3250 3251 static const VMStateDescription vmstate_virtio_device_endian = { 3252 .name = "virtio/device_endian", 3253 .version_id = 1, 3254 .minimum_version_id = 1, 3255 .needed = &virtio_device_endian_needed, 3256 .fields = (VMStateField[]) { 3257 VMSTATE_UINT8(device_endian, VirtIODevice), 3258 VMSTATE_END_OF_LIST() 3259 } 3260 }; 3261 3262 static const VMStateDescription vmstate_virtio_64bit_features = { 3263 .name = "virtio/64bit_features", 3264 .version_id = 1, 3265 .minimum_version_id = 1, 3266 .needed = &virtio_64bit_features_needed, 3267 .fields = (VMStateField[]) { 3268 VMSTATE_UINT64(guest_features, VirtIODevice), 3269 VMSTATE_END_OF_LIST() 3270 } 3271 }; 3272 3273 static const VMStateDescription vmstate_virtio_broken = { 3274 .name = "virtio/broken", 3275 .version_id = 1, 3276 .minimum_version_id = 1, 3277 .needed = &virtio_broken_needed, 3278 .fields = (VMStateField[]) { 3279 VMSTATE_BOOL(broken, VirtIODevice), 3280 VMSTATE_END_OF_LIST() 3281 } 3282 }; 3283 3284 static const VMStateDescription vmstate_virtio_started = { 3285 .name = "virtio/started", 3286 .version_id = 1, 3287 .minimum_version_id = 1, 3288 .needed = &virtio_started_needed, 3289 .fields = (VMStateField[]) { 3290 VMSTATE_BOOL(started, VirtIODevice), 3291 VMSTATE_END_OF_LIST() 3292 } 3293 }; 3294 3295 static const VMStateDescription vmstate_virtio_disabled = { 3296 .name = "virtio/disabled", 3297 .version_id = 1, 3298 .minimum_version_id = 1, 3299 .needed = &virtio_disabled_needed, 3300 .fields = (VMStateField[]) { 3301 VMSTATE_BOOL(disabled, VirtIODevice), 3302 VMSTATE_END_OF_LIST() 3303 } 3304 }; 3305 3306 static const VMStateDescription vmstate_virtio = { 3307 .name = "virtio", 3308 .version_id = 1, 3309 .minimum_version_id = 1, 3310 .fields = (VMStateField[]) { 3311 VMSTATE_END_OF_LIST() 3312 }, 3313 .subsections = (const VMStateDescription*[]) { 3314 &vmstate_virtio_device_endian, 3315 &vmstate_virtio_64bit_features, 3316 &vmstate_virtio_virtqueues, 3317 &vmstate_virtio_ringsize, 3318 &vmstate_virtio_broken, 3319 &vmstate_virtio_extra_state, 3320 &vmstate_virtio_started, 3321 &vmstate_virtio_packed_virtqueues, 3322 &vmstate_virtio_disabled, 3323 NULL 3324 } 3325 }; 3326 3327 int virtio_save(VirtIODevice *vdev, QEMUFile *f) 3328 { 3329 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3330 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3331 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 3332 uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff); 3333 int i; 3334 3335 if (k->save_config) { 3336 k->save_config(qbus->parent, f); 3337 } 3338 3339 qemu_put_8s(f, &vdev->status); 3340 qemu_put_8s(f, &vdev->isr); 3341 qemu_put_be16s(f, &vdev->queue_sel); 3342 qemu_put_be32s(f, &guest_features_lo); 3343 qemu_put_be32(f, vdev->config_len); 3344 qemu_put_buffer(f, vdev->config, vdev->config_len); 3345 3346 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 3347 if (vdev->vq[i].vring.num == 0) 3348 break; 3349 } 3350 3351 qemu_put_be32(f, i); 3352 3353 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 3354 if (vdev->vq[i].vring.num == 0) 3355 break; 3356 3357 qemu_put_be32(f, vdev->vq[i].vring.num); 3358 if (k->has_variable_vring_alignment) { 3359 qemu_put_be32(f, vdev->vq[i].vring.align); 3360 } 3361 /* 3362 * Save desc now, the rest of the ring addresses are saved in 3363 * subsections for VIRTIO-1 devices. 3364 */ 3365 qemu_put_be64(f, vdev->vq[i].vring.desc); 3366 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx); 3367 if (k->save_queue) { 3368 k->save_queue(qbus->parent, i, f); 3369 } 3370 } 3371 3372 if (vdc->save != NULL) { 3373 vdc->save(vdev, f); 3374 } 3375 3376 if (vdc->vmsd) { 3377 int ret = vmstate_save_state(f, vdc->vmsd, vdev, NULL); 3378 if (ret) { 3379 return ret; 3380 } 3381 } 3382 3383 /* Subsections */ 3384 return vmstate_save_state(f, &vmstate_virtio, vdev, NULL); 3385 } 3386 3387 /* A wrapper for use as a VMState .put function */ 3388 static int virtio_device_put(QEMUFile *f, void *opaque, size_t size, 3389 const VMStateField *field, JSONWriter *vmdesc) 3390 { 3391 return virtio_save(VIRTIO_DEVICE(opaque), f); 3392 } 3393 3394 /* A wrapper for use as a VMState .get function */ 3395 static int virtio_device_get(QEMUFile *f, void *opaque, size_t size, 3396 const VMStateField *field) 3397 { 3398 VirtIODevice *vdev = VIRTIO_DEVICE(opaque); 3399 DeviceClass *dc = DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev)); 3400 3401 return virtio_load(vdev, f, dc->vmsd->version_id); 3402 } 3403 3404 const VMStateInfo virtio_vmstate_info = { 3405 .name = "virtio", 3406 .get = virtio_device_get, 3407 .put = virtio_device_put, 3408 }; 3409 3410 static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val) 3411 { 3412 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 3413 bool bad = (val & ~(vdev->host_features)) != 0; 3414 3415 val &= vdev->host_features; 3416 if (k->set_features) { 3417 k->set_features(vdev, val); 3418 } 3419 vdev->guest_features = val; 3420 return bad ? -1 : 0; 3421 } 3422 3423 int virtio_set_features(VirtIODevice *vdev, uint64_t val) 3424 { 3425 int ret; 3426 /* 3427 * The driver must not attempt to set features after feature negotiation 3428 * has finished. 3429 */ 3430 if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) { 3431 return -EINVAL; 3432 } 3433 3434 if (val & (1ull << VIRTIO_F_BAD_FEATURE)) { 3435 qemu_log_mask(LOG_GUEST_ERROR, 3436 "%s: guest driver for %s has enabled UNUSED(30) feature bit!\n", 3437 __func__, vdev->name); 3438 } 3439 3440 ret = virtio_set_features_nocheck(vdev, val); 3441 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 3442 /* VIRTIO_RING_F_EVENT_IDX changes the size of the caches. */ 3443 int i; 3444 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 3445 if (vdev->vq[i].vring.num != 0) { 3446 virtio_init_region_cache(vdev, i); 3447 } 3448 } 3449 } 3450 if (!ret) { 3451 if (!virtio_device_started(vdev, vdev->status) && 3452 !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 3453 vdev->start_on_kick = true; 3454 } 3455 } 3456 return ret; 3457 } 3458 3459 size_t virtio_get_config_size(const VirtIOConfigSizeParams *params, 3460 uint64_t host_features) 3461 { 3462 size_t config_size = params->min_size; 3463 const VirtIOFeature *feature_sizes = params->feature_sizes; 3464 size_t i; 3465 3466 for (i = 0; feature_sizes[i].flags != 0; i++) { 3467 if (host_features & feature_sizes[i].flags) { 3468 config_size = MAX(feature_sizes[i].end, config_size); 3469 } 3470 } 3471 3472 assert(config_size <= params->max_size); 3473 return config_size; 3474 } 3475 3476 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id) 3477 { 3478 int i, ret; 3479 int32_t config_len; 3480 uint32_t num; 3481 uint32_t features; 3482 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3483 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3484 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 3485 3486 /* 3487 * We poison the endianness to ensure it does not get used before 3488 * subsections have been loaded. 3489 */ 3490 vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN; 3491 3492 if (k->load_config) { 3493 ret = k->load_config(qbus->parent, f); 3494 if (ret) 3495 return ret; 3496 } 3497 3498 qemu_get_8s(f, &vdev->status); 3499 qemu_get_8s(f, &vdev->isr); 3500 qemu_get_be16s(f, &vdev->queue_sel); 3501 if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) { 3502 return -1; 3503 } 3504 qemu_get_be32s(f, &features); 3505 3506 /* 3507 * Temporarily set guest_features low bits - needed by 3508 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS 3509 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ. 3510 * 3511 * Note: devices should always test host features in future - don't create 3512 * new dependencies like this. 3513 */ 3514 vdev->guest_features = features; 3515 3516 config_len = qemu_get_be32(f); 3517 3518 /* 3519 * There are cases where the incoming config can be bigger or smaller 3520 * than what we have; so load what we have space for, and skip 3521 * any excess that's in the stream. 3522 */ 3523 qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len)); 3524 3525 while (config_len > vdev->config_len) { 3526 qemu_get_byte(f); 3527 config_len--; 3528 } 3529 3530 num = qemu_get_be32(f); 3531 3532 if (num > VIRTIO_QUEUE_MAX) { 3533 error_report("Invalid number of virtqueues: 0x%x", num); 3534 return -1; 3535 } 3536 3537 for (i = 0; i < num; i++) { 3538 vdev->vq[i].vring.num = qemu_get_be32(f); 3539 if (k->has_variable_vring_alignment) { 3540 vdev->vq[i].vring.align = qemu_get_be32(f); 3541 } 3542 vdev->vq[i].vring.desc = qemu_get_be64(f); 3543 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx); 3544 vdev->vq[i].signalled_used_valid = false; 3545 vdev->vq[i].notification = true; 3546 3547 if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) { 3548 error_report("VQ %d address 0x0 " 3549 "inconsistent with Host index 0x%x", 3550 i, vdev->vq[i].last_avail_idx); 3551 return -1; 3552 } 3553 if (k->load_queue) { 3554 ret = k->load_queue(qbus->parent, i, f); 3555 if (ret) 3556 return ret; 3557 } 3558 } 3559 3560 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); 3561 3562 if (vdc->load != NULL) { 3563 ret = vdc->load(vdev, f, version_id); 3564 if (ret) { 3565 return ret; 3566 } 3567 } 3568 3569 if (vdc->vmsd) { 3570 ret = vmstate_load_state(f, vdc->vmsd, vdev, version_id); 3571 if (ret) { 3572 return ret; 3573 } 3574 } 3575 3576 /* Subsections */ 3577 ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1); 3578 if (ret) { 3579 return ret; 3580 } 3581 3582 if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) { 3583 vdev->device_endian = virtio_default_endian(); 3584 } 3585 3586 if (virtio_64bit_features_needed(vdev)) { 3587 /* 3588 * Subsection load filled vdev->guest_features. Run them 3589 * through virtio_set_features to sanity-check them against 3590 * host_features. 3591 */ 3592 uint64_t features64 = vdev->guest_features; 3593 if (virtio_set_features_nocheck(vdev, features64) < 0) { 3594 error_report("Features 0x%" PRIx64 " unsupported. " 3595 "Allowed features: 0x%" PRIx64, 3596 features64, vdev->host_features); 3597 return -1; 3598 } 3599 } else { 3600 if (virtio_set_features_nocheck(vdev, features) < 0) { 3601 error_report("Features 0x%x unsupported. " 3602 "Allowed features: 0x%" PRIx64, 3603 features, vdev->host_features); 3604 return -1; 3605 } 3606 } 3607 3608 if (!virtio_device_started(vdev, vdev->status) && 3609 !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 3610 vdev->start_on_kick = true; 3611 } 3612 3613 RCU_READ_LOCK_GUARD(); 3614 for (i = 0; i < num; i++) { 3615 if (vdev->vq[i].vring.desc) { 3616 uint16_t nheads; 3617 3618 /* 3619 * VIRTIO-1 devices migrate desc, used, and avail ring addresses so 3620 * only the region cache needs to be set up. Legacy devices need 3621 * to calculate used and avail ring addresses based on the desc 3622 * address. 3623 */ 3624 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 3625 virtio_init_region_cache(vdev, i); 3626 } else { 3627 virtio_queue_update_rings(vdev, i); 3628 } 3629 3630 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3631 vdev->vq[i].shadow_avail_idx = vdev->vq[i].last_avail_idx; 3632 vdev->vq[i].shadow_avail_wrap_counter = 3633 vdev->vq[i].last_avail_wrap_counter; 3634 continue; 3635 } 3636 3637 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx; 3638 /* Check it isn't doing strange things with descriptor numbers. */ 3639 if (nheads > vdev->vq[i].vring.num) { 3640 virtio_error(vdev, "VQ %d size 0x%x Guest index 0x%x " 3641 "inconsistent with Host index 0x%x: delta 0x%x", 3642 i, vdev->vq[i].vring.num, 3643 vring_avail_idx(&vdev->vq[i]), 3644 vdev->vq[i].last_avail_idx, nheads); 3645 vdev->vq[i].used_idx = 0; 3646 vdev->vq[i].shadow_avail_idx = 0; 3647 vdev->vq[i].inuse = 0; 3648 continue; 3649 } 3650 vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]); 3651 vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]); 3652 3653 /* 3654 * Some devices migrate VirtQueueElements that have been popped 3655 * from the avail ring but not yet returned to the used ring. 3656 * Since max ring size < UINT16_MAX it's safe to use modulo 3657 * UINT16_MAX + 1 subtraction. 3658 */ 3659 vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx - 3660 vdev->vq[i].used_idx); 3661 if (vdev->vq[i].inuse > vdev->vq[i].vring.num) { 3662 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - " 3663 "used_idx 0x%x", 3664 i, vdev->vq[i].vring.num, 3665 vdev->vq[i].last_avail_idx, 3666 vdev->vq[i].used_idx); 3667 return -1; 3668 } 3669 } 3670 } 3671 3672 if (vdc->post_load) { 3673 ret = vdc->post_load(vdev); 3674 if (ret) { 3675 return ret; 3676 } 3677 } 3678 3679 return 0; 3680 } 3681 3682 void virtio_cleanup(VirtIODevice *vdev) 3683 { 3684 qemu_del_vm_change_state_handler(vdev->vmstate); 3685 } 3686 3687 static void virtio_vmstate_change(void *opaque, bool running, RunState state) 3688 { 3689 VirtIODevice *vdev = opaque; 3690 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3691 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3692 bool backend_run = running && virtio_device_started(vdev, vdev->status); 3693 vdev->vm_running = running; 3694 3695 if (backend_run) { 3696 virtio_set_status(vdev, vdev->status); 3697 } 3698 3699 if (k->vmstate_change) { 3700 k->vmstate_change(qbus->parent, backend_run); 3701 } 3702 3703 if (!backend_run) { 3704 virtio_set_status(vdev, vdev->status); 3705 } 3706 } 3707 3708 void virtio_instance_init_common(Object *proxy_obj, void *data, 3709 size_t vdev_size, const char *vdev_name) 3710 { 3711 DeviceState *vdev = data; 3712 3713 object_initialize_child_with_props(proxy_obj, "virtio-backend", vdev, 3714 vdev_size, vdev_name, &error_abort, 3715 NULL); 3716 qdev_alias_all_properties(vdev, proxy_obj); 3717 } 3718 3719 void virtio_init(VirtIODevice *vdev, uint16_t device_id, size_t config_size) 3720 { 3721 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3722 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3723 int i; 3724 int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0; 3725 3726 if (nvectors) { 3727 vdev->vector_queues = 3728 g_malloc0(sizeof(*vdev->vector_queues) * nvectors); 3729 } 3730 3731 vdev->start_on_kick = false; 3732 vdev->started = false; 3733 vdev->vhost_started = false; 3734 vdev->device_id = device_id; 3735 vdev->status = 0; 3736 qatomic_set(&vdev->isr, 0); 3737 vdev->queue_sel = 0; 3738 vdev->config_vector = VIRTIO_NO_VECTOR; 3739 vdev->vq = g_new0(VirtQueue, VIRTIO_QUEUE_MAX); 3740 vdev->vm_running = runstate_is_running(); 3741 vdev->broken = false; 3742 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 3743 vdev->vq[i].vector = VIRTIO_NO_VECTOR; 3744 vdev->vq[i].vdev = vdev; 3745 vdev->vq[i].queue_index = i; 3746 vdev->vq[i].host_notifier_enabled = false; 3747 } 3748 3749 vdev->name = virtio_id_to_name(device_id); 3750 vdev->config_len = config_size; 3751 if (vdev->config_len) { 3752 vdev->config = g_malloc0(config_size); 3753 } else { 3754 vdev->config = NULL; 3755 } 3756 vdev->vmstate = qdev_add_vm_change_state_handler(DEVICE(vdev), 3757 virtio_vmstate_change, vdev); 3758 vdev->device_endian = virtio_default_endian(); 3759 vdev->use_guest_notifier_mask = true; 3760 } 3761 3762 /* 3763 * Only devices that have already been around prior to defining the virtio 3764 * standard support legacy mode; this includes devices not specified in the 3765 * standard. All newer devices conform to the virtio standard only. 3766 */ 3767 bool virtio_legacy_allowed(VirtIODevice *vdev) 3768 { 3769 switch (vdev->device_id) { 3770 case VIRTIO_ID_NET: 3771 case VIRTIO_ID_BLOCK: 3772 case VIRTIO_ID_CONSOLE: 3773 case VIRTIO_ID_RNG: 3774 case VIRTIO_ID_BALLOON: 3775 case VIRTIO_ID_RPMSG: 3776 case VIRTIO_ID_SCSI: 3777 case VIRTIO_ID_9P: 3778 case VIRTIO_ID_RPROC_SERIAL: 3779 case VIRTIO_ID_CAIF: 3780 return true; 3781 default: 3782 return false; 3783 } 3784 } 3785 3786 bool virtio_legacy_check_disabled(VirtIODevice *vdev) 3787 { 3788 return vdev->disable_legacy_check; 3789 } 3790 3791 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n) 3792 { 3793 return vdev->vq[n].vring.desc; 3794 } 3795 3796 bool virtio_queue_enabled_legacy(VirtIODevice *vdev, int n) 3797 { 3798 return virtio_queue_get_desc_addr(vdev, n) != 0; 3799 } 3800 3801 bool virtio_queue_enabled(VirtIODevice *vdev, int n) 3802 { 3803 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 3804 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 3805 3806 if (k->queue_enabled) { 3807 return k->queue_enabled(qbus->parent, n); 3808 } 3809 return virtio_queue_enabled_legacy(vdev, n); 3810 } 3811 3812 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n) 3813 { 3814 return vdev->vq[n].vring.avail; 3815 } 3816 3817 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n) 3818 { 3819 return vdev->vq[n].vring.used; 3820 } 3821 3822 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n) 3823 { 3824 return sizeof(VRingDesc) * vdev->vq[n].vring.num; 3825 } 3826 3827 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n) 3828 { 3829 int s; 3830 3831 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3832 return sizeof(struct VRingPackedDescEvent); 3833 } 3834 3835 s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 3836 return offsetof(VRingAvail, ring) + 3837 sizeof(uint16_t) * vdev->vq[n].vring.num + s; 3838 } 3839 3840 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n) 3841 { 3842 int s; 3843 3844 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3845 return sizeof(struct VRingPackedDescEvent); 3846 } 3847 3848 s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 3849 return offsetof(VRingUsed, ring) + 3850 sizeof(VRingUsedElem) * vdev->vq[n].vring.num + s; 3851 } 3852 3853 static unsigned int virtio_queue_packed_get_last_avail_idx(VirtIODevice *vdev, 3854 int n) 3855 { 3856 unsigned int avail, used; 3857 3858 avail = vdev->vq[n].last_avail_idx; 3859 avail |= ((uint16_t)vdev->vq[n].last_avail_wrap_counter) << 15; 3860 3861 used = vdev->vq[n].used_idx; 3862 used |= ((uint16_t)vdev->vq[n].used_wrap_counter) << 15; 3863 3864 return avail | used << 16; 3865 } 3866 3867 static uint16_t virtio_queue_split_get_last_avail_idx(VirtIODevice *vdev, 3868 int n) 3869 { 3870 return vdev->vq[n].last_avail_idx; 3871 } 3872 3873 unsigned int virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n) 3874 { 3875 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3876 return virtio_queue_packed_get_last_avail_idx(vdev, n); 3877 } else { 3878 return virtio_queue_split_get_last_avail_idx(vdev, n); 3879 } 3880 } 3881 3882 static void virtio_queue_packed_set_last_avail_idx(VirtIODevice *vdev, 3883 int n, unsigned int idx) 3884 { 3885 struct VirtQueue *vq = &vdev->vq[n]; 3886 3887 vq->last_avail_idx = vq->shadow_avail_idx = idx & 0x7fff; 3888 vq->last_avail_wrap_counter = 3889 vq->shadow_avail_wrap_counter = !!(idx & 0x8000); 3890 idx >>= 16; 3891 vq->used_idx = idx & 0x7ffff; 3892 vq->used_wrap_counter = !!(idx & 0x8000); 3893 } 3894 3895 static void virtio_queue_split_set_last_avail_idx(VirtIODevice *vdev, 3896 int n, unsigned int idx) 3897 { 3898 vdev->vq[n].last_avail_idx = idx; 3899 vdev->vq[n].shadow_avail_idx = idx; 3900 } 3901 3902 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, 3903 unsigned int idx) 3904 { 3905 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3906 virtio_queue_packed_set_last_avail_idx(vdev, n, idx); 3907 } else { 3908 virtio_queue_split_set_last_avail_idx(vdev, n, idx); 3909 } 3910 } 3911 3912 static void virtio_queue_packed_restore_last_avail_idx(VirtIODevice *vdev, 3913 int n) 3914 { 3915 /* We don't have a reference like avail idx in shared memory */ 3916 return; 3917 } 3918 3919 static void virtio_queue_split_restore_last_avail_idx(VirtIODevice *vdev, 3920 int n) 3921 { 3922 RCU_READ_LOCK_GUARD(); 3923 if (vdev->vq[n].vring.desc) { 3924 vdev->vq[n].last_avail_idx = vring_used_idx(&vdev->vq[n]); 3925 vdev->vq[n].shadow_avail_idx = vdev->vq[n].last_avail_idx; 3926 } 3927 } 3928 3929 void virtio_queue_restore_last_avail_idx(VirtIODevice *vdev, int n) 3930 { 3931 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3932 virtio_queue_packed_restore_last_avail_idx(vdev, n); 3933 } else { 3934 virtio_queue_split_restore_last_avail_idx(vdev, n); 3935 } 3936 } 3937 3938 static void virtio_queue_packed_update_used_idx(VirtIODevice *vdev, int n) 3939 { 3940 /* used idx was updated through set_last_avail_idx() */ 3941 return; 3942 } 3943 3944 static void virtio_split_packed_update_used_idx(VirtIODevice *vdev, int n) 3945 { 3946 RCU_READ_LOCK_GUARD(); 3947 if (vdev->vq[n].vring.desc) { 3948 vdev->vq[n].used_idx = vring_used_idx(&vdev->vq[n]); 3949 } 3950 } 3951 3952 void virtio_queue_update_used_idx(VirtIODevice *vdev, int n) 3953 { 3954 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 3955 return virtio_queue_packed_update_used_idx(vdev, n); 3956 } else { 3957 return virtio_split_packed_update_used_idx(vdev, n); 3958 } 3959 } 3960 3961 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n) 3962 { 3963 vdev->vq[n].signalled_used_valid = false; 3964 } 3965 3966 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n) 3967 { 3968 return vdev->vq + n; 3969 } 3970 3971 uint16_t virtio_get_queue_index(VirtQueue *vq) 3972 { 3973 return vq->queue_index; 3974 } 3975 3976 static void virtio_queue_guest_notifier_read(EventNotifier *n) 3977 { 3978 VirtQueue *vq = container_of(n, VirtQueue, guest_notifier); 3979 if (event_notifier_test_and_clear(n)) { 3980 virtio_irq(vq); 3981 } 3982 } 3983 3984 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign, 3985 bool with_irqfd) 3986 { 3987 if (assign && !with_irqfd) { 3988 event_notifier_set_handler(&vq->guest_notifier, 3989 virtio_queue_guest_notifier_read); 3990 } else { 3991 event_notifier_set_handler(&vq->guest_notifier, NULL); 3992 } 3993 if (!assign) { 3994 /* Test and clear notifier before closing it, 3995 * in case poll callback didn't have time to run. */ 3996 virtio_queue_guest_notifier_read(&vq->guest_notifier); 3997 } 3998 } 3999 4000 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq) 4001 { 4002 return &vq->guest_notifier; 4003 } 4004 4005 static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier *n) 4006 { 4007 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 4008 4009 virtio_queue_set_notification(vq, 0); 4010 } 4011 4012 static bool virtio_queue_host_notifier_aio_poll(void *opaque) 4013 { 4014 EventNotifier *n = opaque; 4015 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 4016 4017 return vq->vring.desc && !virtio_queue_empty(vq); 4018 } 4019 4020 static void virtio_queue_host_notifier_aio_poll_ready(EventNotifier *n) 4021 { 4022 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 4023 4024 virtio_queue_notify_vq(vq); 4025 } 4026 4027 static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n) 4028 { 4029 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 4030 4031 /* Caller polls once more after this to catch requests that race with us */ 4032 virtio_queue_set_notification(vq, 1); 4033 } 4034 4035 void virtio_queue_aio_attach_host_notifier(VirtQueue *vq, AioContext *ctx) 4036 { 4037 aio_set_event_notifier(ctx, &vq->host_notifier, true, 4038 virtio_queue_host_notifier_read, 4039 virtio_queue_host_notifier_aio_poll, 4040 virtio_queue_host_notifier_aio_poll_ready); 4041 aio_set_event_notifier_poll(ctx, &vq->host_notifier, 4042 virtio_queue_host_notifier_aio_poll_begin, 4043 virtio_queue_host_notifier_aio_poll_end); 4044 } 4045 4046 /* 4047 * Same as virtio_queue_aio_attach_host_notifier() but without polling. Use 4048 * this for rx virtqueues and similar cases where the virtqueue handler 4049 * function does not pop all elements. When the virtqueue is left non-empty 4050 * polling consumes CPU cycles and should not be used. 4051 */ 4052 void virtio_queue_aio_attach_host_notifier_no_poll(VirtQueue *vq, AioContext *ctx) 4053 { 4054 aio_set_event_notifier(ctx, &vq->host_notifier, true, 4055 virtio_queue_host_notifier_read, 4056 NULL, NULL); 4057 } 4058 4059 void virtio_queue_aio_detach_host_notifier(VirtQueue *vq, AioContext *ctx) 4060 { 4061 aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL, NULL, NULL); 4062 /* Test and clear notifier before after disabling event, 4063 * in case poll callback didn't have time to run. */ 4064 virtio_queue_host_notifier_read(&vq->host_notifier); 4065 } 4066 4067 void virtio_queue_host_notifier_read(EventNotifier *n) 4068 { 4069 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 4070 if (event_notifier_test_and_clear(n)) { 4071 virtio_queue_notify_vq(vq); 4072 } 4073 } 4074 4075 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq) 4076 { 4077 return &vq->host_notifier; 4078 } 4079 4080 void virtio_queue_set_host_notifier_enabled(VirtQueue *vq, bool enabled) 4081 { 4082 vq->host_notifier_enabled = enabled; 4083 } 4084 4085 int virtio_queue_set_host_notifier_mr(VirtIODevice *vdev, int n, 4086 MemoryRegion *mr, bool assign) 4087 { 4088 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 4089 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 4090 4091 if (k->set_host_notifier_mr) { 4092 return k->set_host_notifier_mr(qbus->parent, n, mr, assign); 4093 } 4094 4095 return -1; 4096 } 4097 4098 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name) 4099 { 4100 g_free(vdev->bus_name); 4101 vdev->bus_name = g_strdup(bus_name); 4102 } 4103 4104 void G_GNUC_PRINTF(2, 3) virtio_error(VirtIODevice *vdev, const char *fmt, ...) 4105 { 4106 va_list ap; 4107 4108 va_start(ap, fmt); 4109 error_vreport(fmt, ap); 4110 va_end(ap); 4111 4112 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 4113 vdev->status = vdev->status | VIRTIO_CONFIG_S_NEEDS_RESET; 4114 virtio_notify_config(vdev); 4115 } 4116 4117 vdev->broken = true; 4118 } 4119 4120 static void virtio_memory_listener_commit(MemoryListener *listener) 4121 { 4122 VirtIODevice *vdev = container_of(listener, VirtIODevice, listener); 4123 int i; 4124 4125 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 4126 if (vdev->vq[i].vring.num == 0) { 4127 break; 4128 } 4129 virtio_init_region_cache(vdev, i); 4130 } 4131 } 4132 4133 static void virtio_device_realize(DeviceState *dev, Error **errp) 4134 { 4135 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 4136 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 4137 Error *err = NULL; 4138 4139 /* Devices should either use vmsd or the load/save methods */ 4140 assert(!vdc->vmsd || !vdc->load); 4141 4142 if (vdc->realize != NULL) { 4143 vdc->realize(dev, &err); 4144 if (err != NULL) { 4145 error_propagate(errp, err); 4146 return; 4147 } 4148 } 4149 4150 virtio_bus_device_plugged(vdev, &err); 4151 if (err != NULL) { 4152 error_propagate(errp, err); 4153 vdc->unrealize(dev); 4154 return; 4155 } 4156 4157 vdev->listener.commit = virtio_memory_listener_commit; 4158 vdev->listener.name = "virtio"; 4159 memory_listener_register(&vdev->listener, vdev->dma_as); 4160 QTAILQ_INSERT_TAIL(&virtio_list, vdev, next); 4161 } 4162 4163 static void virtio_device_unrealize(DeviceState *dev) 4164 { 4165 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 4166 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 4167 4168 memory_listener_unregister(&vdev->listener); 4169 virtio_bus_device_unplugged(vdev); 4170 4171 if (vdc->unrealize != NULL) { 4172 vdc->unrealize(dev); 4173 } 4174 4175 QTAILQ_REMOVE(&virtio_list, vdev, next); 4176 g_free(vdev->bus_name); 4177 vdev->bus_name = NULL; 4178 } 4179 4180 static void virtio_device_free_virtqueues(VirtIODevice *vdev) 4181 { 4182 int i; 4183 if (!vdev->vq) { 4184 return; 4185 } 4186 4187 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 4188 if (vdev->vq[i].vring.num == 0) { 4189 break; 4190 } 4191 virtio_virtqueue_reset_region_cache(&vdev->vq[i]); 4192 } 4193 g_free(vdev->vq); 4194 } 4195 4196 static void virtio_device_instance_finalize(Object *obj) 4197 { 4198 VirtIODevice *vdev = VIRTIO_DEVICE(obj); 4199 4200 virtio_device_free_virtqueues(vdev); 4201 4202 g_free(vdev->config); 4203 g_free(vdev->vector_queues); 4204 } 4205 4206 static Property virtio_properties[] = { 4207 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features), 4208 DEFINE_PROP_BOOL("use-started", VirtIODevice, use_started, true), 4209 DEFINE_PROP_BOOL("use-disabled-flag", VirtIODevice, use_disabled_flag, true), 4210 DEFINE_PROP_BOOL("x-disable-legacy-check", VirtIODevice, 4211 disable_legacy_check, false), 4212 DEFINE_PROP_END_OF_LIST(), 4213 }; 4214 4215 static int virtio_device_start_ioeventfd_impl(VirtIODevice *vdev) 4216 { 4217 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev))); 4218 int i, n, r, err; 4219 4220 /* 4221 * Batch all the host notifiers in a single transaction to avoid 4222 * quadratic time complexity in address_space_update_ioeventfds(). 4223 */ 4224 memory_region_transaction_begin(); 4225 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 4226 VirtQueue *vq = &vdev->vq[n]; 4227 if (!virtio_queue_get_num(vdev, n)) { 4228 continue; 4229 } 4230 r = virtio_bus_set_host_notifier(qbus, n, true); 4231 if (r < 0) { 4232 err = r; 4233 goto assign_error; 4234 } 4235 event_notifier_set_handler(&vq->host_notifier, 4236 virtio_queue_host_notifier_read); 4237 } 4238 4239 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 4240 /* Kick right away to begin processing requests already in vring */ 4241 VirtQueue *vq = &vdev->vq[n]; 4242 if (!vq->vring.num) { 4243 continue; 4244 } 4245 event_notifier_set(&vq->host_notifier); 4246 } 4247 memory_region_transaction_commit(); 4248 return 0; 4249 4250 assign_error: 4251 i = n; /* save n for a second iteration after transaction is committed. */ 4252 while (--n >= 0) { 4253 VirtQueue *vq = &vdev->vq[n]; 4254 if (!virtio_queue_get_num(vdev, n)) { 4255 continue; 4256 } 4257 4258 event_notifier_set_handler(&vq->host_notifier, NULL); 4259 r = virtio_bus_set_host_notifier(qbus, n, false); 4260 assert(r >= 0); 4261 } 4262 /* 4263 * The transaction expects the ioeventfds to be open when it 4264 * commits. Do it now, before the cleanup loop. 4265 */ 4266 memory_region_transaction_commit(); 4267 4268 while (--i >= 0) { 4269 if (!virtio_queue_get_num(vdev, i)) { 4270 continue; 4271 } 4272 virtio_bus_cleanup_host_notifier(qbus, i); 4273 } 4274 return err; 4275 } 4276 4277 int virtio_device_start_ioeventfd(VirtIODevice *vdev) 4278 { 4279 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 4280 VirtioBusState *vbus = VIRTIO_BUS(qbus); 4281 4282 return virtio_bus_start_ioeventfd(vbus); 4283 } 4284 4285 static void virtio_device_stop_ioeventfd_impl(VirtIODevice *vdev) 4286 { 4287 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev))); 4288 int n, r; 4289 4290 /* 4291 * Batch all the host notifiers in a single transaction to avoid 4292 * quadratic time complexity in address_space_update_ioeventfds(). 4293 */ 4294 memory_region_transaction_begin(); 4295 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 4296 VirtQueue *vq = &vdev->vq[n]; 4297 4298 if (!virtio_queue_get_num(vdev, n)) { 4299 continue; 4300 } 4301 event_notifier_set_handler(&vq->host_notifier, NULL); 4302 r = virtio_bus_set_host_notifier(qbus, n, false); 4303 assert(r >= 0); 4304 } 4305 /* 4306 * The transaction expects the ioeventfds to be open when it 4307 * commits. Do it now, before the cleanup loop. 4308 */ 4309 memory_region_transaction_commit(); 4310 4311 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 4312 if (!virtio_queue_get_num(vdev, n)) { 4313 continue; 4314 } 4315 virtio_bus_cleanup_host_notifier(qbus, n); 4316 } 4317 } 4318 4319 int virtio_device_grab_ioeventfd(VirtIODevice *vdev) 4320 { 4321 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 4322 VirtioBusState *vbus = VIRTIO_BUS(qbus); 4323 4324 return virtio_bus_grab_ioeventfd(vbus); 4325 } 4326 4327 void virtio_device_release_ioeventfd(VirtIODevice *vdev) 4328 { 4329 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 4330 VirtioBusState *vbus = VIRTIO_BUS(qbus); 4331 4332 virtio_bus_release_ioeventfd(vbus); 4333 } 4334 4335 static void virtio_device_class_init(ObjectClass *klass, void *data) 4336 { 4337 /* Set the default value here. */ 4338 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 4339 DeviceClass *dc = DEVICE_CLASS(klass); 4340 4341 dc->realize = virtio_device_realize; 4342 dc->unrealize = virtio_device_unrealize; 4343 dc->bus_type = TYPE_VIRTIO_BUS; 4344 device_class_set_props(dc, virtio_properties); 4345 vdc->start_ioeventfd = virtio_device_start_ioeventfd_impl; 4346 vdc->stop_ioeventfd = virtio_device_stop_ioeventfd_impl; 4347 4348 vdc->legacy_features |= VIRTIO_LEGACY_FEATURES; 4349 4350 QTAILQ_INIT(&virtio_list); 4351 } 4352 4353 bool virtio_device_ioeventfd_enabled(VirtIODevice *vdev) 4354 { 4355 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 4356 VirtioBusState *vbus = VIRTIO_BUS(qbus); 4357 4358 return virtio_bus_ioeventfd_enabled(vbus); 4359 } 4360 4361 VirtioInfoList *qmp_x_query_virtio(Error **errp) 4362 { 4363 VirtioInfoList *list = NULL; 4364 VirtioInfoList *node; 4365 VirtIODevice *vdev; 4366 4367 QTAILQ_FOREACH(vdev, &virtio_list, next) { 4368 DeviceState *dev = DEVICE(vdev); 4369 Error *err = NULL; 4370 QObject *obj = qmp_qom_get(dev->canonical_path, "realized", &err); 4371 4372 if (err == NULL) { 4373 GString *is_realized = qobject_to_json_pretty(obj, true); 4374 /* virtio device is NOT realized, remove it from list */ 4375 if (!strncmp(is_realized->str, "false", 4)) { 4376 QTAILQ_REMOVE(&virtio_list, vdev, next); 4377 } else { 4378 node = g_new0(VirtioInfoList, 1); 4379 node->value = g_new(VirtioInfo, 1); 4380 node->value->path = g_strdup(dev->canonical_path); 4381 node->value->name = g_strdup(vdev->name); 4382 QAPI_LIST_PREPEND(list, node->value); 4383 } 4384 g_string_free(is_realized, true); 4385 } 4386 qobject_unref(obj); 4387 } 4388 4389 return list; 4390 } 4391 4392 static VirtIODevice *virtio_device_find(const char *path) 4393 { 4394 VirtIODevice *vdev; 4395 4396 QTAILQ_FOREACH(vdev, &virtio_list, next) { 4397 DeviceState *dev = DEVICE(vdev); 4398 4399 if (strcmp(dev->canonical_path, path) != 0) { 4400 continue; 4401 } 4402 4403 Error *err = NULL; 4404 QObject *obj = qmp_qom_get(dev->canonical_path, "realized", &err); 4405 if (err == NULL) { 4406 GString *is_realized = qobject_to_json_pretty(obj, true); 4407 /* virtio device is NOT realized, remove it from list */ 4408 if (!strncmp(is_realized->str, "false", 4)) { 4409 g_string_free(is_realized, true); 4410 qobject_unref(obj); 4411 QTAILQ_REMOVE(&virtio_list, vdev, next); 4412 return NULL; 4413 } 4414 g_string_free(is_realized, true); 4415 } else { 4416 /* virtio device doesn't exist in QOM tree */ 4417 QTAILQ_REMOVE(&virtio_list, vdev, next); 4418 qobject_unref(obj); 4419 return NULL; 4420 } 4421 /* device exists in QOM tree & is realized */ 4422 qobject_unref(obj); 4423 return vdev; 4424 } 4425 return NULL; 4426 } 4427 4428 #define CONVERT_FEATURES(type, map, is_status, bitmap) \ 4429 ({ \ 4430 type *list = NULL; \ 4431 type *node; \ 4432 for (i = 0; map[i].virtio_bit != -1; i++) { \ 4433 if (is_status) { \ 4434 bit = map[i].virtio_bit; \ 4435 } \ 4436 else { \ 4437 bit = 1ULL << map[i].virtio_bit; \ 4438 } \ 4439 if ((bitmap & bit) == 0) { \ 4440 continue; \ 4441 } \ 4442 node = g_new0(type, 1); \ 4443 node->value = g_strdup(map[i].feature_desc); \ 4444 node->next = list; \ 4445 list = node; \ 4446 bitmap ^= bit; \ 4447 } \ 4448 list; \ 4449 }) 4450 4451 static VirtioDeviceStatus *qmp_decode_status(uint8_t bitmap) 4452 { 4453 VirtioDeviceStatus *status; 4454 uint8_t bit; 4455 int i; 4456 4457 status = g_new0(VirtioDeviceStatus, 1); 4458 status->statuses = CONVERT_FEATURES(strList, virtio_config_status_map, 4459 1, bitmap); 4460 status->has_unknown_statuses = bitmap != 0; 4461 if (status->has_unknown_statuses) { 4462 status->unknown_statuses = bitmap; 4463 } 4464 4465 return status; 4466 } 4467 4468 static VhostDeviceProtocols *qmp_decode_protocols(uint64_t bitmap) 4469 { 4470 VhostDeviceProtocols *vhu_protocols; 4471 uint64_t bit; 4472 int i; 4473 4474 vhu_protocols = g_new0(VhostDeviceProtocols, 1); 4475 vhu_protocols->protocols = 4476 CONVERT_FEATURES(strList, 4477 vhost_user_protocol_map, 0, bitmap); 4478 vhu_protocols->has_unknown_protocols = bitmap != 0; 4479 if (vhu_protocols->has_unknown_protocols) { 4480 vhu_protocols->unknown_protocols = bitmap; 4481 } 4482 4483 return vhu_protocols; 4484 } 4485 4486 static VirtioDeviceFeatures *qmp_decode_features(uint16_t device_id, 4487 uint64_t bitmap) 4488 { 4489 VirtioDeviceFeatures *features; 4490 uint64_t bit; 4491 int i; 4492 4493 features = g_new0(VirtioDeviceFeatures, 1); 4494 features->has_dev_features = true; 4495 4496 /* transport features */ 4497 features->transports = CONVERT_FEATURES(strList, virtio_transport_map, 0, 4498 bitmap); 4499 4500 /* device features */ 4501 switch (device_id) { 4502 #ifdef CONFIG_VIRTIO_SERIAL 4503 case VIRTIO_ID_CONSOLE: 4504 features->dev_features = 4505 CONVERT_FEATURES(strList, virtio_serial_feature_map, 0, bitmap); 4506 break; 4507 #endif 4508 #ifdef CONFIG_VIRTIO_BLK 4509 case VIRTIO_ID_BLOCK: 4510 features->dev_features = 4511 CONVERT_FEATURES(strList, virtio_blk_feature_map, 0, bitmap); 4512 break; 4513 #endif 4514 #ifdef CONFIG_VIRTIO_GPU 4515 case VIRTIO_ID_GPU: 4516 features->dev_features = 4517 CONVERT_FEATURES(strList, virtio_gpu_feature_map, 0, bitmap); 4518 break; 4519 #endif 4520 #ifdef CONFIG_VIRTIO_NET 4521 case VIRTIO_ID_NET: 4522 features->dev_features = 4523 CONVERT_FEATURES(strList, virtio_net_feature_map, 0, bitmap); 4524 break; 4525 #endif 4526 #ifdef CONFIG_VIRTIO_SCSI 4527 case VIRTIO_ID_SCSI: 4528 features->dev_features = 4529 CONVERT_FEATURES(strList, virtio_scsi_feature_map, 0, bitmap); 4530 break; 4531 #endif 4532 #ifdef CONFIG_VIRTIO_BALLOON 4533 case VIRTIO_ID_BALLOON: 4534 features->dev_features = 4535 CONVERT_FEATURES(strList, virtio_balloon_feature_map, 0, bitmap); 4536 break; 4537 #endif 4538 #ifdef CONFIG_VIRTIO_IOMMU 4539 case VIRTIO_ID_IOMMU: 4540 features->dev_features = 4541 CONVERT_FEATURES(strList, virtio_iommu_feature_map, 0, bitmap); 4542 break; 4543 #endif 4544 #ifdef CONFIG_VIRTIO_INPUT 4545 case VIRTIO_ID_INPUT: 4546 features->dev_features = 4547 CONVERT_FEATURES(strList, virtio_input_feature_map, 0, bitmap); 4548 break; 4549 #endif 4550 #ifdef CONFIG_VHOST_USER_FS 4551 case VIRTIO_ID_FS: 4552 features->dev_features = 4553 CONVERT_FEATURES(strList, virtio_fs_feature_map, 0, bitmap); 4554 break; 4555 #endif 4556 #ifdef CONFIG_VHOST_VSOCK 4557 case VIRTIO_ID_VSOCK: 4558 features->dev_features = 4559 CONVERT_FEATURES(strList, virtio_vsock_feature_map, 0, bitmap); 4560 break; 4561 #endif 4562 #ifdef CONFIG_VIRTIO_CRYPTO 4563 case VIRTIO_ID_CRYPTO: 4564 features->dev_features = 4565 CONVERT_FEATURES(strList, virtio_crypto_feature_map, 0, bitmap); 4566 break; 4567 #endif 4568 #ifdef CONFIG_VIRTIO_MEM 4569 case VIRTIO_ID_MEM: 4570 features->dev_features = 4571 CONVERT_FEATURES(strList, virtio_mem_feature_map, 0, bitmap); 4572 break; 4573 #endif 4574 #ifdef CONFIG_VIRTIO_I2C_ADAPTER 4575 case VIRTIO_ID_I2C_ADAPTER: 4576 features->dev_features = 4577 CONVERT_FEATURES(strList, virtio_i2c_feature_map, 0, bitmap); 4578 break; 4579 #endif 4580 #ifdef CONFIG_VIRTIO_RNG 4581 case VIRTIO_ID_RNG: 4582 features->dev_features = 4583 CONVERT_FEATURES(strList, virtio_rng_feature_map, 0, bitmap); 4584 break; 4585 #endif 4586 /* No features */ 4587 case VIRTIO_ID_9P: 4588 case VIRTIO_ID_PMEM: 4589 case VIRTIO_ID_IOMEM: 4590 case VIRTIO_ID_RPMSG: 4591 case VIRTIO_ID_CLOCK: 4592 case VIRTIO_ID_MAC80211_WLAN: 4593 case VIRTIO_ID_MAC80211_HWSIM: 4594 case VIRTIO_ID_RPROC_SERIAL: 4595 case VIRTIO_ID_MEMORY_BALLOON: 4596 case VIRTIO_ID_CAIF: 4597 case VIRTIO_ID_SIGNAL_DIST: 4598 case VIRTIO_ID_PSTORE: 4599 case VIRTIO_ID_SOUND: 4600 case VIRTIO_ID_BT: 4601 case VIRTIO_ID_RPMB: 4602 case VIRTIO_ID_VIDEO_ENCODER: 4603 case VIRTIO_ID_VIDEO_DECODER: 4604 case VIRTIO_ID_SCMI: 4605 case VIRTIO_ID_NITRO_SEC_MOD: 4606 case VIRTIO_ID_WATCHDOG: 4607 case VIRTIO_ID_CAN: 4608 case VIRTIO_ID_DMABUF: 4609 case VIRTIO_ID_PARAM_SERV: 4610 case VIRTIO_ID_AUDIO_POLICY: 4611 case VIRTIO_ID_GPIO: 4612 break; 4613 default: 4614 g_assert_not_reached(); 4615 } 4616 4617 features->has_unknown_dev_features = bitmap != 0; 4618 if (features->has_unknown_dev_features) { 4619 features->unknown_dev_features = bitmap; 4620 } 4621 4622 return features; 4623 } 4624 4625 VirtioStatus *qmp_x_query_virtio_status(const char *path, Error **errp) 4626 { 4627 VirtIODevice *vdev; 4628 VirtioStatus *status; 4629 4630 vdev = virtio_device_find(path); 4631 if (vdev == NULL) { 4632 error_setg(errp, "Path %s is not a VirtIODevice", path); 4633 return NULL; 4634 } 4635 4636 status = g_new0(VirtioStatus, 1); 4637 status->name = g_strdup(vdev->name); 4638 status->device_id = vdev->device_id; 4639 status->vhost_started = vdev->vhost_started; 4640 status->guest_features = qmp_decode_features(vdev->device_id, 4641 vdev->guest_features); 4642 status->host_features = qmp_decode_features(vdev->device_id, 4643 vdev->host_features); 4644 status->backend_features = qmp_decode_features(vdev->device_id, 4645 vdev->backend_features); 4646 4647 switch (vdev->device_endian) { 4648 case VIRTIO_DEVICE_ENDIAN_LITTLE: 4649 status->device_endian = g_strdup("little"); 4650 break; 4651 case VIRTIO_DEVICE_ENDIAN_BIG: 4652 status->device_endian = g_strdup("big"); 4653 break; 4654 default: 4655 status->device_endian = g_strdup("unknown"); 4656 break; 4657 } 4658 4659 status->num_vqs = virtio_get_num_queues(vdev); 4660 status->status = qmp_decode_status(vdev->status); 4661 status->isr = vdev->isr; 4662 status->queue_sel = vdev->queue_sel; 4663 status->vm_running = vdev->vm_running; 4664 status->broken = vdev->broken; 4665 status->disabled = vdev->disabled; 4666 status->use_started = vdev->use_started; 4667 status->started = vdev->started; 4668 status->start_on_kick = vdev->start_on_kick; 4669 status->disable_legacy_check = vdev->disable_legacy_check; 4670 status->bus_name = g_strdup(vdev->bus_name); 4671 status->use_guest_notifier_mask = vdev->use_guest_notifier_mask; 4672 status->has_vhost_dev = vdev->vhost_started; 4673 4674 if (vdev->vhost_started) { 4675 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 4676 struct vhost_dev *hdev = vdc->get_vhost(vdev); 4677 4678 status->vhost_dev = g_new0(VhostStatus, 1); 4679 status->vhost_dev->n_mem_sections = hdev->n_mem_sections; 4680 status->vhost_dev->n_tmp_sections = hdev->n_tmp_sections; 4681 status->vhost_dev->nvqs = hdev->nvqs; 4682 status->vhost_dev->vq_index = hdev->vq_index; 4683 status->vhost_dev->features = 4684 qmp_decode_features(vdev->device_id, hdev->features); 4685 status->vhost_dev->acked_features = 4686 qmp_decode_features(vdev->device_id, hdev->acked_features); 4687 status->vhost_dev->backend_features = 4688 qmp_decode_features(vdev->device_id, hdev->backend_features); 4689 status->vhost_dev->protocol_features = 4690 qmp_decode_protocols(hdev->protocol_features); 4691 status->vhost_dev->max_queues = hdev->max_queues; 4692 status->vhost_dev->backend_cap = hdev->backend_cap; 4693 status->vhost_dev->log_enabled = hdev->log_enabled; 4694 status->vhost_dev->log_size = hdev->log_size; 4695 } 4696 4697 return status; 4698 } 4699 4700 VirtVhostQueueStatus *qmp_x_query_virtio_vhost_queue_status(const char *path, 4701 uint16_t queue, 4702 Error **errp) 4703 { 4704 VirtIODevice *vdev; 4705 VirtVhostQueueStatus *status; 4706 4707 vdev = virtio_device_find(path); 4708 if (vdev == NULL) { 4709 error_setg(errp, "Path %s is not a VirtIODevice", path); 4710 return NULL; 4711 } 4712 4713 if (!vdev->vhost_started) { 4714 error_setg(errp, "Error: vhost device has not started yet"); 4715 return NULL; 4716 } 4717 4718 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 4719 struct vhost_dev *hdev = vdc->get_vhost(vdev); 4720 4721 if (queue < hdev->vq_index || queue >= hdev->vq_index + hdev->nvqs) { 4722 error_setg(errp, "Invalid vhost virtqueue number %d", queue); 4723 return NULL; 4724 } 4725 4726 status = g_new0(VirtVhostQueueStatus, 1); 4727 status->name = g_strdup(vdev->name); 4728 status->kick = hdev->vqs[queue].kick; 4729 status->call = hdev->vqs[queue].call; 4730 status->desc = (uintptr_t)hdev->vqs[queue].desc; 4731 status->avail = (uintptr_t)hdev->vqs[queue].avail; 4732 status->used = (uintptr_t)hdev->vqs[queue].used; 4733 status->num = hdev->vqs[queue].num; 4734 status->desc_phys = hdev->vqs[queue].desc_phys; 4735 status->desc_size = hdev->vqs[queue].desc_size; 4736 status->avail_phys = hdev->vqs[queue].avail_phys; 4737 status->avail_size = hdev->vqs[queue].avail_size; 4738 status->used_phys = hdev->vqs[queue].used_phys; 4739 status->used_size = hdev->vqs[queue].used_size; 4740 4741 return status; 4742 } 4743 4744 VirtQueueStatus *qmp_x_query_virtio_queue_status(const char *path, 4745 uint16_t queue, 4746 Error **errp) 4747 { 4748 VirtIODevice *vdev; 4749 VirtQueueStatus *status; 4750 4751 vdev = virtio_device_find(path); 4752 if (vdev == NULL) { 4753 error_setg(errp, "Path %s is not a VirtIODevice", path); 4754 return NULL; 4755 } 4756 4757 if (queue >= VIRTIO_QUEUE_MAX || !virtio_queue_get_num(vdev, queue)) { 4758 error_setg(errp, "Invalid virtqueue number %d", queue); 4759 return NULL; 4760 } 4761 4762 status = g_new0(VirtQueueStatus, 1); 4763 status->name = g_strdup(vdev->name); 4764 status->queue_index = vdev->vq[queue].queue_index; 4765 status->inuse = vdev->vq[queue].inuse; 4766 status->vring_num = vdev->vq[queue].vring.num; 4767 status->vring_num_default = vdev->vq[queue].vring.num_default; 4768 status->vring_align = vdev->vq[queue].vring.align; 4769 status->vring_desc = vdev->vq[queue].vring.desc; 4770 status->vring_avail = vdev->vq[queue].vring.avail; 4771 status->vring_used = vdev->vq[queue].vring.used; 4772 status->used_idx = vdev->vq[queue].used_idx; 4773 status->signalled_used = vdev->vq[queue].signalled_used; 4774 status->signalled_used_valid = vdev->vq[queue].signalled_used_valid; 4775 4776 if (vdev->vhost_started) { 4777 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 4778 struct vhost_dev *hdev = vdc->get_vhost(vdev); 4779 4780 /* check if vq index exists for vhost as well */ 4781 if (queue >= hdev->vq_index && queue < hdev->vq_index + hdev->nvqs) { 4782 status->has_last_avail_idx = true; 4783 4784 int vhost_vq_index = 4785 hdev->vhost_ops->vhost_get_vq_index(hdev, queue); 4786 struct vhost_vring_state state = { 4787 .index = vhost_vq_index, 4788 }; 4789 4790 status->last_avail_idx = 4791 hdev->vhost_ops->vhost_get_vring_base(hdev, &state); 4792 } 4793 } else { 4794 status->has_shadow_avail_idx = true; 4795 status->has_last_avail_idx = true; 4796 status->last_avail_idx = vdev->vq[queue].last_avail_idx; 4797 status->shadow_avail_idx = vdev->vq[queue].shadow_avail_idx; 4798 } 4799 4800 return status; 4801 } 4802 4803 static strList *qmp_decode_vring_desc_flags(uint16_t flags) 4804 { 4805 strList *list = NULL; 4806 strList *node; 4807 int i; 4808 4809 struct { 4810 uint16_t flag; 4811 const char *value; 4812 } map[] = { 4813 { VRING_DESC_F_NEXT, "next" }, 4814 { VRING_DESC_F_WRITE, "write" }, 4815 { VRING_DESC_F_INDIRECT, "indirect" }, 4816 { 1 << VRING_PACKED_DESC_F_AVAIL, "avail" }, 4817 { 1 << VRING_PACKED_DESC_F_USED, "used" }, 4818 { 0, "" } 4819 }; 4820 4821 for (i = 0; map[i].flag; i++) { 4822 if ((map[i].flag & flags) == 0) { 4823 continue; 4824 } 4825 node = g_malloc0(sizeof(strList)); 4826 node->value = g_strdup(map[i].value); 4827 node->next = list; 4828 list = node; 4829 } 4830 4831 return list; 4832 } 4833 4834 VirtioQueueElement *qmp_x_query_virtio_queue_element(const char *path, 4835 uint16_t queue, 4836 bool has_index, 4837 uint16_t index, 4838 Error **errp) 4839 { 4840 VirtIODevice *vdev; 4841 VirtQueue *vq; 4842 VirtioQueueElement *element = NULL; 4843 4844 vdev = virtio_device_find(path); 4845 if (vdev == NULL) { 4846 error_setg(errp, "Path %s is not a VirtIO device", path); 4847 return NULL; 4848 } 4849 4850 if (queue >= VIRTIO_QUEUE_MAX || !virtio_queue_get_num(vdev, queue)) { 4851 error_setg(errp, "Invalid virtqueue number %d", queue); 4852 return NULL; 4853 } 4854 vq = &vdev->vq[queue]; 4855 4856 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 4857 error_setg(errp, "Packed ring not supported"); 4858 return NULL; 4859 } else { 4860 unsigned int head, i, max; 4861 VRingMemoryRegionCaches *caches; 4862 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 4863 MemoryRegionCache *desc_cache; 4864 VRingDesc desc; 4865 VirtioRingDescList *list = NULL; 4866 VirtioRingDescList *node; 4867 int rc; int ndescs; 4868 4869 RCU_READ_LOCK_GUARD(); 4870 4871 max = vq->vring.num; 4872 4873 if (!has_index) { 4874 head = vring_avail_ring(vq, vq->last_avail_idx % vq->vring.num); 4875 } else { 4876 head = vring_avail_ring(vq, index % vq->vring.num); 4877 } 4878 i = head; 4879 4880 caches = vring_get_region_caches(vq); 4881 if (!caches) { 4882 error_setg(errp, "Region caches not initialized"); 4883 return NULL; 4884 } 4885 if (caches->desc.len < max * sizeof(VRingDesc)) { 4886 error_setg(errp, "Cannot map descriptor ring"); 4887 return NULL; 4888 } 4889 4890 desc_cache = &caches->desc; 4891 vring_split_desc_read(vdev, &desc, desc_cache, i); 4892 if (desc.flags & VRING_DESC_F_INDIRECT) { 4893 int64_t len; 4894 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, 4895 desc.addr, desc.len, false); 4896 desc_cache = &indirect_desc_cache; 4897 if (len < desc.len) { 4898 error_setg(errp, "Cannot map indirect buffer"); 4899 goto done; 4900 } 4901 4902 max = desc.len / sizeof(VRingDesc); 4903 i = 0; 4904 vring_split_desc_read(vdev, &desc, desc_cache, i); 4905 } 4906 4907 element = g_new0(VirtioQueueElement, 1); 4908 element->avail = g_new0(VirtioRingAvail, 1); 4909 element->used = g_new0(VirtioRingUsed, 1); 4910 element->name = g_strdup(vdev->name); 4911 element->index = head; 4912 element->avail->flags = vring_avail_flags(vq); 4913 element->avail->idx = vring_avail_idx(vq); 4914 element->avail->ring = head; 4915 element->used->flags = vring_used_flags(vq); 4916 element->used->idx = vring_used_idx(vq); 4917 ndescs = 0; 4918 4919 do { 4920 /* A buggy driver may produce an infinite loop */ 4921 if (ndescs >= max) { 4922 break; 4923 } 4924 node = g_new0(VirtioRingDescList, 1); 4925 node->value = g_new0(VirtioRingDesc, 1); 4926 node->value->addr = desc.addr; 4927 node->value->len = desc.len; 4928 node->value->flags = qmp_decode_vring_desc_flags(desc.flags); 4929 node->next = list; 4930 list = node; 4931 4932 ndescs++; 4933 rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, 4934 max, &i); 4935 } while (rc == VIRTQUEUE_READ_DESC_MORE); 4936 element->descs = list; 4937 done: 4938 address_space_cache_destroy(&indirect_desc_cache); 4939 } 4940 4941 return element; 4942 } 4943 4944 static const TypeInfo virtio_device_info = { 4945 .name = TYPE_VIRTIO_DEVICE, 4946 .parent = TYPE_DEVICE, 4947 .instance_size = sizeof(VirtIODevice), 4948 .class_init = virtio_device_class_init, 4949 .instance_finalize = virtio_device_instance_finalize, 4950 .abstract = true, 4951 .class_size = sizeof(VirtioDeviceClass), 4952 }; 4953 4954 static void virtio_register_types(void) 4955 { 4956 type_register_static(&virtio_device_info); 4957 } 4958 4959 type_init(virtio_register_types) 4960