1 /*
2 * libqos VIRTIO 1.0 PCI driver
3 *
4 * Copyright (c) 2019 Red Hat, Inc
5 *
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
8 */
9
10 #include "qemu/osdep.h"
11 #include "standard-headers/linux/pci_regs.h"
12 #include "standard-headers/linux/virtio_pci.h"
13 #include "standard-headers/linux/virtio_config.h"
14 #include "virtio-pci-modern.h"
15
config_readb(QVirtioDevice * d,uint64_t addr)16 static uint8_t config_readb(QVirtioDevice *d, uint64_t addr)
17 {
18 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
19 return qpci_io_readb(dev->pdev, dev->bar, dev->device_cfg_offset + addr);
20 }
21
config_readw(QVirtioDevice * d,uint64_t addr)22 static uint16_t config_readw(QVirtioDevice *d, uint64_t addr)
23 {
24 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
25 return qpci_io_readw(dev->pdev, dev->bar, dev->device_cfg_offset + addr);
26 }
27
config_readl(QVirtioDevice * d,uint64_t addr)28 static uint32_t config_readl(QVirtioDevice *d, uint64_t addr)
29 {
30 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
31 return qpci_io_readl(dev->pdev, dev->bar, dev->device_cfg_offset + addr);
32 }
33
config_readq(QVirtioDevice * d,uint64_t addr)34 static uint64_t config_readq(QVirtioDevice *d, uint64_t addr)
35 {
36 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
37 return qpci_io_readq(dev->pdev, dev->bar, dev->device_cfg_offset + addr);
38 }
39
get_features(QVirtioDevice * d)40 static uint64_t get_features(QVirtioDevice *d)
41 {
42 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
43 uint64_t lo, hi;
44
45 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
46 offsetof(struct virtio_pci_common_cfg,
47 device_feature_select),
48 0);
49 lo = qpci_io_readl(dev->pdev, dev->bar, dev->common_cfg_offset +
50 offsetof(struct virtio_pci_common_cfg, device_feature));
51
52 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
53 offsetof(struct virtio_pci_common_cfg,
54 device_feature_select),
55 1);
56 hi = qpci_io_readl(dev->pdev, dev->bar, dev->common_cfg_offset +
57 offsetof(struct virtio_pci_common_cfg, device_feature));
58
59 return (hi << 32) | lo;
60 }
61
set_features(QVirtioDevice * d,uint64_t features)62 static void set_features(QVirtioDevice *d, uint64_t features)
63 {
64 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
65
66 /* Drivers must enable VIRTIO 1.0 or else use the Legacy interface */
67 g_assert_cmphex(features & (1ull << VIRTIO_F_VERSION_1), !=, 0);
68
69 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
70 offsetof(struct virtio_pci_common_cfg,
71 guest_feature_select),
72 0);
73 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
74 offsetof(struct virtio_pci_common_cfg,
75 guest_feature),
76 features);
77 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
78 offsetof(struct virtio_pci_common_cfg,
79 guest_feature_select),
80 1);
81 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
82 offsetof(struct virtio_pci_common_cfg,
83 guest_feature),
84 features >> 32);
85 }
86
get_guest_features(QVirtioDevice * d)87 static uint64_t get_guest_features(QVirtioDevice *d)
88 {
89 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
90 uint64_t lo, hi;
91
92 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
93 offsetof(struct virtio_pci_common_cfg,
94 guest_feature_select),
95 0);
96 lo = qpci_io_readl(dev->pdev, dev->bar, dev->common_cfg_offset +
97 offsetof(struct virtio_pci_common_cfg, guest_feature));
98
99 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
100 offsetof(struct virtio_pci_common_cfg,
101 guest_feature_select),
102 1);
103 hi = qpci_io_readl(dev->pdev, dev->bar, dev->common_cfg_offset +
104 offsetof(struct virtio_pci_common_cfg, guest_feature));
105
106 return (hi << 32) | lo;
107 }
108
get_status(QVirtioDevice * d)109 static uint8_t get_status(QVirtioDevice *d)
110 {
111 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
112
113 return qpci_io_readb(dev->pdev, dev->bar, dev->common_cfg_offset +
114 offsetof(struct virtio_pci_common_cfg,
115 device_status));
116 }
117
set_status(QVirtioDevice * d,uint8_t status)118 static void set_status(QVirtioDevice *d, uint8_t status)
119 {
120 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
121
122 return qpci_io_writeb(dev->pdev, dev->bar, dev->common_cfg_offset +
123 offsetof(struct virtio_pci_common_cfg,
124 device_status),
125 status);
126 }
127
get_msix_status(QVirtioPCIDevice * dev,uint32_t msix_entry,uint32_t msix_addr,uint32_t msix_data)128 static bool get_msix_status(QVirtioPCIDevice *dev, uint32_t msix_entry,
129 uint32_t msix_addr, uint32_t msix_data)
130 {
131 uint32_t data;
132
133 g_assert_cmpint(msix_entry, !=, -1);
134 if (qpci_msix_masked(dev->pdev, msix_entry)) {
135 /* No ISR checking should be done if masked, but read anyway */
136 return qpci_msix_pending(dev->pdev, msix_entry);
137 }
138
139 data = qtest_readl(dev->pdev->bus->qts, msix_addr);
140 if (data == msix_data) {
141 qtest_writel(dev->pdev->bus->qts, msix_addr, 0);
142 return true;
143 } else {
144 return false;
145 }
146 }
147
get_queue_isr_status(QVirtioDevice * d,QVirtQueue * vq)148 static bool get_queue_isr_status(QVirtioDevice *d, QVirtQueue *vq)
149 {
150 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
151
152 if (dev->pdev->msix_enabled) {
153 QVirtQueuePCI *vqpci = container_of(vq, QVirtQueuePCI, vq);
154
155 return get_msix_status(dev, vqpci->msix_entry, vqpci->msix_addr,
156 vqpci->msix_data);
157 }
158
159 return qpci_io_readb(dev->pdev, dev->bar, dev->isr_cfg_offset) & 1;
160 }
161
get_config_isr_status(QVirtioDevice * d)162 static bool get_config_isr_status(QVirtioDevice *d)
163 {
164 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
165
166 if (dev->pdev->msix_enabled) {
167 return get_msix_status(dev, dev->config_msix_entry,
168 dev->config_msix_addr, dev->config_msix_data);
169 }
170
171 return qpci_io_readb(dev->pdev, dev->bar, dev->isr_cfg_offset) & 2;
172 }
173
wait_config_isr_status(QVirtioDevice * d,gint64 timeout_us)174 static void wait_config_isr_status(QVirtioDevice *d, gint64 timeout_us)
175 {
176 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
177 gint64 start_time = g_get_monotonic_time();
178
179 do {
180 g_assert(g_get_monotonic_time() - start_time <= timeout_us);
181 qtest_clock_step(dev->pdev->bus->qts, 100);
182 } while (!get_config_isr_status(d));
183 }
184
queue_select(QVirtioDevice * d,uint16_t index)185 static void queue_select(QVirtioDevice *d, uint16_t index)
186 {
187 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
188
189 qpci_io_writew(dev->pdev, dev->bar, dev->common_cfg_offset +
190 offsetof(struct virtio_pci_common_cfg, queue_select),
191 index);
192 }
193
get_queue_size(QVirtioDevice * d)194 static uint16_t get_queue_size(QVirtioDevice *d)
195 {
196 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
197
198 return qpci_io_readw(dev->pdev, dev->bar, dev->common_cfg_offset +
199 offsetof(struct virtio_pci_common_cfg, queue_size));
200 }
201
set_queue_address(QVirtioDevice * d,QVirtQueue * vq)202 static void set_queue_address(QVirtioDevice *d, QVirtQueue *vq)
203 {
204 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
205
206 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
207 offsetof(struct virtio_pci_common_cfg, queue_desc_lo),
208 vq->desc);
209 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
210 offsetof(struct virtio_pci_common_cfg, queue_desc_hi),
211 vq->desc >> 32);
212
213 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
214 offsetof(struct virtio_pci_common_cfg, queue_avail_lo),
215 vq->avail);
216 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
217 offsetof(struct virtio_pci_common_cfg, queue_avail_hi),
218 vq->avail >> 32);
219
220 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
221 offsetof(struct virtio_pci_common_cfg, queue_used_lo),
222 vq->used);
223 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
224 offsetof(struct virtio_pci_common_cfg, queue_used_hi),
225 vq->used >> 32);
226 }
227
virtqueue_setup(QVirtioDevice * d,QGuestAllocator * alloc,uint16_t index)228 static QVirtQueue *virtqueue_setup(QVirtioDevice *d, QGuestAllocator *alloc,
229 uint16_t index)
230 {
231 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
232 QVirtQueue *vq;
233 QVirtQueuePCI *vqpci;
234 uint16_t notify_off;
235
236 vq = qvirtio_pci_virtqueue_setup_common(d, alloc, index);
237 vqpci = container_of(vq, QVirtQueuePCI, vq);
238
239 notify_off = qpci_io_readw(dev->pdev, dev->bar, dev->common_cfg_offset +
240 offsetof(struct virtio_pci_common_cfg,
241 queue_notify_off));
242
243 vqpci->notify_offset = dev->notify_cfg_offset +
244 notify_off * dev->notify_off_multiplier;
245
246 qpci_io_writew(dev->pdev, dev->bar, dev->common_cfg_offset +
247 offsetof(struct virtio_pci_common_cfg, queue_enable), 1);
248
249 return vq;
250 }
251
virtqueue_kick(QVirtioDevice * d,QVirtQueue * vq)252 static void virtqueue_kick(QVirtioDevice *d, QVirtQueue *vq)
253 {
254 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
255 QVirtQueuePCI *vqpci = container_of(vq, QVirtQueuePCI, vq);
256
257 qpci_io_writew(dev->pdev, dev->bar, vqpci->notify_offset, vq->index);
258 }
259
260 static const QVirtioBus qvirtio_pci_virtio_1 = {
261 .config_readb = config_readb,
262 .config_readw = config_readw,
263 .config_readl = config_readl,
264 .config_readq = config_readq,
265 .get_features = get_features,
266 .set_features = set_features,
267 .get_guest_features = get_guest_features,
268 .get_status = get_status,
269 .set_status = set_status,
270 .get_queue_isr_status = get_queue_isr_status,
271 .wait_config_isr_status = wait_config_isr_status,
272 .queue_select = queue_select,
273 .get_queue_size = get_queue_size,
274 .set_queue_address = set_queue_address,
275 .virtqueue_setup = virtqueue_setup,
276 .virtqueue_cleanup = qvirtio_pci_virtqueue_cleanup_common,
277 .virtqueue_kick = virtqueue_kick,
278 };
279
set_config_vector(QVirtioPCIDevice * d,uint16_t entry)280 static void set_config_vector(QVirtioPCIDevice *d, uint16_t entry)
281 {
282 uint16_t vector;
283
284 qpci_io_writew(d->pdev, d->bar, d->common_cfg_offset +
285 offsetof(struct virtio_pci_common_cfg, msix_config), entry);
286 vector = qpci_io_readw(d->pdev, d->bar, d->common_cfg_offset +
287 offsetof(struct virtio_pci_common_cfg,
288 msix_config));
289 g_assert_cmphex(vector, !=, VIRTIO_MSI_NO_VECTOR);
290 }
291
set_queue_vector(QVirtioPCIDevice * d,uint16_t vq_idx,uint16_t entry)292 static void set_queue_vector(QVirtioPCIDevice *d, uint16_t vq_idx,
293 uint16_t entry)
294 {
295 uint16_t vector;
296
297 queue_select(&d->vdev, vq_idx);
298 qpci_io_writew(d->pdev, d->bar, d->common_cfg_offset +
299 offsetof(struct virtio_pci_common_cfg, queue_msix_vector),
300 entry);
301 vector = qpci_io_readw(d->pdev, d->bar, d->common_cfg_offset +
302 offsetof(struct virtio_pci_common_cfg,
303 queue_msix_vector));
304 g_assert_cmphex(vector, !=, VIRTIO_MSI_NO_VECTOR);
305 }
306
307 static const QVirtioPCIMSIXOps qvirtio_pci_msix_ops_virtio_1 = {
308 .set_config_vector = set_config_vector,
309 .set_queue_vector = set_queue_vector,
310 };
311
probe_device_type(QVirtioPCIDevice * dev)312 static bool probe_device_type(QVirtioPCIDevice *dev)
313 {
314 uint16_t vendor_id;
315 uint16_t device_id;
316
317 /* "Drivers MUST match devices with the PCI Vendor ID 0x1AF4" */
318 vendor_id = qpci_config_readw(dev->pdev, PCI_VENDOR_ID);
319 if (vendor_id != 0x1af4) {
320 return false;
321 }
322
323 /*
324 * "Any PCI device with ... PCI Device ID 0x1000 through 0x107F inclusive
325 * is a virtio device"
326 */
327 device_id = qpci_config_readw(dev->pdev, PCI_DEVICE_ID);
328 if (device_id < 0x1000 || device_id > 0x107f) {
329 return false;
330 }
331
332 /*
333 * "Devices MAY utilize a Transitional PCI Device ID range, 0x1000 to
334 * 0x103F depending on the device type"
335 */
336 if (device_id < 0x1040) {
337 /*
338 * "Transitional devices MUST have the PCI Subsystem Device ID matching
339 * the Virtio Device ID"
340 */
341 dev->vdev.device_type = qpci_config_readw(dev->pdev, PCI_SUBSYSTEM_ID);
342 } else {
343 /*
344 * "The PCI Device ID is calculated by adding 0x1040 to the Virtio
345 * Device ID"
346 */
347 dev->vdev.device_type = device_id - 0x1040;
348 }
349
350 return true;
351 }
352
353 /* Find the first VIRTIO 1.0 PCI structure for a given type */
find_structure(QVirtioPCIDevice * dev,uint8_t cfg_type,uint8_t * bar,uint32_t * offset,uint32_t * length,uint8_t * cfg_addr)354 static bool find_structure(QVirtioPCIDevice *dev, uint8_t cfg_type,
355 uint8_t *bar, uint32_t *offset, uint32_t *length,
356 uint8_t *cfg_addr)
357 {
358 uint8_t addr = 0;
359
360 while ((addr = qpci_find_capability(dev->pdev, PCI_CAP_ID_VNDR,
361 addr)) != 0) {
362 uint8_t type;
363
364 type = qpci_config_readb(dev->pdev,
365 addr + offsetof(struct virtio_pci_cap, cfg_type));
366 if (type != cfg_type) {
367 continue;
368 }
369
370 *bar = qpci_config_readb(dev->pdev,
371 addr + offsetof(struct virtio_pci_cap, bar));
372 *offset = qpci_config_readl(dev->pdev,
373 addr + offsetof(struct virtio_pci_cap, offset));
374 *length = qpci_config_readl(dev->pdev,
375 addr + offsetof(struct virtio_pci_cap, length));
376 if (cfg_addr) {
377 *cfg_addr = addr;
378 }
379
380 return true;
381 }
382
383 return false;
384 }
385
probe_device_layout(QVirtioPCIDevice * dev)386 static bool probe_device_layout(QVirtioPCIDevice *dev)
387 {
388 uint8_t bar;
389 uint8_t cfg_addr;
390 uint32_t length;
391
392 /*
393 * Due to the qpci_iomap() API we only support devices that put all
394 * structures in the same PCI BAR. Luckily this is true with QEMU.
395 */
396
397 if (!find_structure(dev, VIRTIO_PCI_CAP_COMMON_CFG, &bar,
398 &dev->common_cfg_offset, &length, NULL)) {
399 return false;
400 }
401 dev->bar_idx = bar;
402
403 if (!find_structure(dev, VIRTIO_PCI_CAP_NOTIFY_CFG, &bar,
404 &dev->notify_cfg_offset, &length, &cfg_addr)) {
405 return false;
406 }
407 g_assert_cmphex(bar, ==, dev->bar_idx);
408
409 dev->notify_off_multiplier = qpci_config_readl(dev->pdev,
410 cfg_addr + offsetof(struct virtio_pci_notify_cap,
411 notify_off_multiplier));
412
413 if (!find_structure(dev, VIRTIO_PCI_CAP_ISR_CFG, &bar,
414 &dev->isr_cfg_offset, &length, NULL)) {
415 return false;
416 }
417 g_assert_cmphex(bar, ==, dev->bar_idx);
418
419 if (!find_structure(dev, VIRTIO_PCI_CAP_DEVICE_CFG, &bar,
420 &dev->device_cfg_offset, &length, NULL)) {
421 return false;
422 }
423 g_assert_cmphex(bar, ==, dev->bar_idx);
424
425 return true;
426 }
427
428 /* Probe a VIRTIO 1.0 device */
qvirtio_pci_init_virtio_1(QVirtioPCIDevice * dev)429 bool qvirtio_pci_init_virtio_1(QVirtioPCIDevice *dev)
430 {
431 if (!probe_device_type(dev)) {
432 return false;
433 }
434
435 if (!probe_device_layout(dev)) {
436 return false;
437 }
438
439 dev->vdev.bus = &qvirtio_pci_virtio_1;
440 dev->msix_ops = &qvirtio_pci_msix_ops_virtio_1;
441 dev->vdev.big_endian = false;
442 return true;
443 }
444