xref: /openbmc/qemu/hw/xen/xen_pt_msi.c (revision a89f364ae8740dfc31b321eed9ee454e996dc3c1)
1  /*
2   * Copyright (c) 2007, Intel Corporation.
3   *
4   * This work is licensed under the terms of the GNU GPL, version 2.  See
5   * the COPYING file in the top-level directory.
6   *
7   * Jiang Yunhong <yunhong.jiang@intel.com>
8   *
9   * This file implements direct PCI assignment to a HVM guest
10   */
11  
12  #include "qemu/osdep.h"
13  
14  #include "hw/xen/xen_backend.h"
15  #include "xen_pt.h"
16  #include "hw/i386/apic-msidef.h"
17  
18  
19  #define XEN_PT_AUTO_ASSIGN -1
20  
21  /* shift count for gflags */
22  #define XEN_PT_GFLAGS_SHIFT_DEST_ID        0
23  #define XEN_PT_GFLAGS_SHIFT_RH             8
24  #define XEN_PT_GFLAGS_SHIFT_DM             9
25  #define XEN_PT_GFLAGSSHIFT_DELIV_MODE     12
26  #define XEN_PT_GFLAGSSHIFT_TRG_MODE       15
27  #define XEN_PT_GFLAGSSHIFT_UNMASKED       16
28  
29  #define latch(fld) latch[PCI_MSIX_ENTRY_##fld / sizeof(uint32_t)]
30  
31  /*
32   * Helpers
33   */
34  
35  static inline uint8_t msi_vector(uint32_t data)
36  {
37      return (data & MSI_DATA_VECTOR_MASK) >> MSI_DATA_VECTOR_SHIFT;
38  }
39  
40  static inline uint8_t msi_dest_id(uint32_t addr)
41  {
42      return (addr & MSI_ADDR_DEST_ID_MASK) >> MSI_ADDR_DEST_ID_SHIFT;
43  }
44  
45  static inline uint32_t msi_ext_dest_id(uint32_t addr_hi)
46  {
47      return addr_hi & 0xffffff00;
48  }
49  
50  static uint32_t msi_gflags(uint32_t data, uint64_t addr)
51  {
52      uint32_t result = 0;
53      int rh, dm, dest_id, deliv_mode, trig_mode;
54  
55      rh = (addr >> MSI_ADDR_REDIRECTION_SHIFT) & 0x1;
56      dm = (addr >> MSI_ADDR_DEST_MODE_SHIFT) & 0x1;
57      dest_id = msi_dest_id(addr);
58      deliv_mode = (data >> MSI_DATA_DELIVERY_MODE_SHIFT) & 0x7;
59      trig_mode = (data >> MSI_DATA_TRIGGER_SHIFT) & 0x1;
60  
61      result = dest_id | (rh << XEN_PT_GFLAGS_SHIFT_RH)
62          | (dm << XEN_PT_GFLAGS_SHIFT_DM)
63          | (deliv_mode << XEN_PT_GFLAGSSHIFT_DELIV_MODE)
64          | (trig_mode << XEN_PT_GFLAGSSHIFT_TRG_MODE);
65  
66      return result;
67  }
68  
69  static inline uint64_t msi_addr64(XenPTMSI *msi)
70  {
71      return (uint64_t)msi->addr_hi << 32 | msi->addr_lo;
72  }
73  
74  static int msi_msix_enable(XenPCIPassthroughState *s,
75                             uint32_t address,
76                             uint16_t flag,
77                             bool enable)
78  {
79      uint16_t val = 0;
80      int rc;
81  
82      if (!address) {
83          return -1;
84      }
85  
86      rc = xen_host_pci_get_word(&s->real_device, address, &val);
87      if (rc) {
88          XEN_PT_ERR(&s->dev, "Failed to read MSI/MSI-X register (0x%x), rc:%d\n",
89                     address, rc);
90          return rc;
91      }
92      if (enable) {
93          val |= flag;
94      } else {
95          val &= ~flag;
96      }
97      rc = xen_host_pci_set_word(&s->real_device, address, val);
98      if (rc) {
99          XEN_PT_ERR(&s->dev, "Failed to write MSI/MSI-X register (0x%x), rc:%d\n",
100                     address, rc);
101      }
102      return rc;
103  }
104  
105  static int msi_msix_setup(XenPCIPassthroughState *s,
106                            uint64_t addr,
107                            uint32_t data,
108                            int *ppirq,
109                            bool is_msix,
110                            int msix_entry,
111                            bool is_not_mapped)
112  {
113      uint8_t gvec = msi_vector(data);
114      int rc = 0;
115  
116      assert((!is_msix && msix_entry == 0) || is_msix);
117  
118      if (xen_is_pirq_msi(data)) {
119          *ppirq = msi_ext_dest_id(addr >> 32) | msi_dest_id(addr);
120          if (!*ppirq) {
121              /* this probably identifies an misconfiguration of the guest,
122               * try the emulated path */
123              *ppirq = XEN_PT_UNASSIGNED_PIRQ;
124          } else {
125              XEN_PT_LOG(&s->dev, "requested pirq %d for MSI%s"
126                         " (vec: %#x, entry: %#x)\n",
127                         *ppirq, is_msix ? "-X" : "", gvec, msix_entry);
128          }
129      }
130  
131      if (is_not_mapped) {
132          uint64_t table_base = 0;
133  
134          if (is_msix) {
135              table_base = s->msix->table_base;
136          }
137  
138          rc = xc_physdev_map_pirq_msi(xen_xc, xen_domid, XEN_PT_AUTO_ASSIGN,
139                                       ppirq, PCI_DEVFN(s->real_device.dev,
140                                                        s->real_device.func),
141                                       s->real_device.bus,
142                                       msix_entry, table_base);
143          if (rc) {
144              XEN_PT_ERR(&s->dev,
145                         "Mapping of MSI%s (err: %i, vec: %#x, entry %#x)\n",
146                         is_msix ? "-X" : "", errno, gvec, msix_entry);
147              return rc;
148          }
149      }
150  
151      return 0;
152  }
153  static int msi_msix_update(XenPCIPassthroughState *s,
154                             uint64_t addr,
155                             uint32_t data,
156                             int pirq,
157                             bool is_msix,
158                             int msix_entry,
159                             int *old_pirq,
160                             bool masked)
161  {
162      PCIDevice *d = &s->dev;
163      uint8_t gvec = msi_vector(data);
164      uint32_t gflags = msi_gflags(data, addr);
165      int rc = 0;
166      uint64_t table_addr = 0;
167  
168      XEN_PT_LOG(d, "Updating MSI%s with pirq %d gvec %#x gflags %#x"
169                 " (entry: %#x)\n",
170                 is_msix ? "-X" : "", pirq, gvec, gflags, msix_entry);
171  
172      if (is_msix) {
173          table_addr = s->msix->mmio_base_addr;
174      }
175  
176      gflags |= masked ? 0 : (1u << XEN_PT_GFLAGSSHIFT_UNMASKED);
177  
178      rc = xc_domain_update_msi_irq(xen_xc, xen_domid, gvec,
179                                    pirq, gflags, table_addr);
180  
181      if (rc) {
182          XEN_PT_ERR(d, "Updating of MSI%s failed. (err: %d)\n",
183                     is_msix ? "-X" : "", errno);
184  
185          if (xc_physdev_unmap_pirq(xen_xc, xen_domid, *old_pirq)) {
186              XEN_PT_ERR(d, "Unmapping of MSI%s pirq %d failed. (err: %d)\n",
187                         is_msix ? "-X" : "", *old_pirq, errno);
188          }
189          *old_pirq = XEN_PT_UNASSIGNED_PIRQ;
190      }
191      return rc;
192  }
193  
194  static int msi_msix_disable(XenPCIPassthroughState *s,
195                              uint64_t addr,
196                              uint32_t data,
197                              int pirq,
198                              bool is_msix,
199                              bool is_binded)
200  {
201      PCIDevice *d = &s->dev;
202      uint8_t gvec = msi_vector(data);
203      uint32_t gflags = msi_gflags(data, addr);
204      int rc = 0;
205  
206      if (pirq == XEN_PT_UNASSIGNED_PIRQ) {
207          return 0;
208      }
209  
210      if (is_binded) {
211          XEN_PT_LOG(d, "Unbind MSI%s with pirq %d, gvec %#x\n",
212                     is_msix ? "-X" : "", pirq, gvec);
213          rc = xc_domain_unbind_msi_irq(xen_xc, xen_domid, gvec, pirq, gflags);
214          if (rc) {
215              XEN_PT_ERR(d, "Unbinding of MSI%s failed. (err: %d, pirq: %d, gvec: %#x)\n",
216                         is_msix ? "-X" : "", errno, pirq, gvec);
217              return rc;
218          }
219      }
220  
221      XEN_PT_LOG(d, "Unmap MSI%s pirq %d\n", is_msix ? "-X" : "", pirq);
222      rc = xc_physdev_unmap_pirq(xen_xc, xen_domid, pirq);
223      if (rc) {
224          XEN_PT_ERR(d, "Unmapping of MSI%s pirq %d failed. (err: %i)\n",
225                     is_msix ? "-X" : "", pirq, errno);
226          return rc;
227      }
228  
229      return 0;
230  }
231  
232  /*
233   * MSI virtualization functions
234   */
235  
236  static int xen_pt_msi_set_enable(XenPCIPassthroughState *s, bool enable)
237  {
238      XEN_PT_LOG(&s->dev, "%s MSI.\n", enable ? "enabling" : "disabling");
239  
240      if (!s->msi) {
241          return -1;
242      }
243  
244      return msi_msix_enable(s, s->msi->ctrl_offset, PCI_MSI_FLAGS_ENABLE,
245                             enable);
246  }
247  
248  /* setup physical msi, but don't enable it */
249  int xen_pt_msi_setup(XenPCIPassthroughState *s)
250  {
251      int pirq = XEN_PT_UNASSIGNED_PIRQ;
252      int rc = 0;
253      XenPTMSI *msi = s->msi;
254  
255      if (msi->initialized) {
256          XEN_PT_ERR(&s->dev,
257                     "Setup physical MSI when it has been properly initialized.\n");
258          return -1;
259      }
260  
261      rc = msi_msix_setup(s, msi_addr64(msi), msi->data, &pirq, false, 0, true);
262      if (rc) {
263          return rc;
264      }
265  
266      if (pirq < 0) {
267          XEN_PT_ERR(&s->dev, "Invalid pirq number: %d.\n", pirq);
268          return -1;
269      }
270  
271      msi->pirq = pirq;
272      XEN_PT_LOG(&s->dev, "MSI mapped with pirq %d.\n", pirq);
273  
274      return 0;
275  }
276  
277  int xen_pt_msi_update(XenPCIPassthroughState *s)
278  {
279      XenPTMSI *msi = s->msi;
280  
281      /* Current MSI emulation in QEMU only supports 1 vector */
282      return msi_msix_update(s, msi_addr64(msi), msi->data, msi->pirq,
283                             false, 0, &msi->pirq, msi->mask & 1);
284  }
285  
286  void xen_pt_msi_disable(XenPCIPassthroughState *s)
287  {
288      XenPTMSI *msi = s->msi;
289  
290      if (!msi) {
291          return;
292      }
293  
294      (void)xen_pt_msi_set_enable(s, false);
295  
296      msi_msix_disable(s, msi_addr64(msi), msi->data, msi->pirq, false,
297                       msi->initialized);
298  
299      /* clear msi info */
300      msi->flags &= ~PCI_MSI_FLAGS_ENABLE;
301      msi->initialized = false;
302      msi->mapped = false;
303      msi->pirq = XEN_PT_UNASSIGNED_PIRQ;
304  }
305  
306  /*
307   * MSI-X virtualization functions
308   */
309  
310  static int msix_set_enable(XenPCIPassthroughState *s, bool enabled)
311  {
312      XEN_PT_LOG(&s->dev, "%s MSI-X.\n", enabled ? "enabling" : "disabling");
313  
314      if (!s->msix) {
315          return -1;
316      }
317  
318      return msi_msix_enable(s, s->msix->ctrl_offset, PCI_MSIX_FLAGS_ENABLE,
319                             enabled);
320  }
321  
322  static int xen_pt_msix_update_one(XenPCIPassthroughState *s, int entry_nr,
323                                    uint32_t vec_ctrl)
324  {
325      XenPTMSIXEntry *entry = NULL;
326      int pirq;
327      int rc;
328  
329      if (entry_nr < 0 || entry_nr >= s->msix->total_entries) {
330          return -EINVAL;
331      }
332  
333      entry = &s->msix->msix_entry[entry_nr];
334  
335      if (!entry->updated) {
336          return 0;
337      }
338  
339      pirq = entry->pirq;
340  
341      /*
342       * Update the entry addr and data to the latest values only when the
343       * entry is masked or they are all masked, as required by the spec.
344       * Addr and data changes while the MSI-X entry is unmasked get deferred
345       * until the next masked -> unmasked transition.
346       */
347      if (pirq == XEN_PT_UNASSIGNED_PIRQ || s->msix->maskall ||
348          (vec_ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT)) {
349          entry->addr = entry->latch(LOWER_ADDR) |
350                        ((uint64_t)entry->latch(UPPER_ADDR) << 32);
351          entry->data = entry->latch(DATA);
352      }
353  
354      rc = msi_msix_setup(s, entry->addr, entry->data, &pirq, true, entry_nr,
355                          entry->pirq == XEN_PT_UNASSIGNED_PIRQ);
356      if (rc) {
357          return rc;
358      }
359      if (entry->pirq == XEN_PT_UNASSIGNED_PIRQ) {
360          entry->pirq = pirq;
361      }
362  
363      rc = msi_msix_update(s, entry->addr, entry->data, pirq, true,
364                           entry_nr, &entry->pirq,
365                           vec_ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT);
366  
367      if (!rc) {
368          entry->updated = false;
369      }
370  
371      return rc;
372  }
373  
374  int xen_pt_msix_update(XenPCIPassthroughState *s)
375  {
376      XenPTMSIX *msix = s->msix;
377      int i;
378  
379      for (i = 0; i < msix->total_entries; i++) {
380          xen_pt_msix_update_one(s, i, msix->msix_entry[i].latch(VECTOR_CTRL));
381      }
382  
383      return 0;
384  }
385  
386  void xen_pt_msix_disable(XenPCIPassthroughState *s)
387  {
388      int i = 0;
389  
390      msix_set_enable(s, false);
391  
392      for (i = 0; i < s->msix->total_entries; i++) {
393          XenPTMSIXEntry *entry = &s->msix->msix_entry[i];
394  
395          msi_msix_disable(s, entry->addr, entry->data, entry->pirq, true, true);
396  
397          /* clear MSI-X info */
398          entry->pirq = XEN_PT_UNASSIGNED_PIRQ;
399          entry->updated = false;
400      }
401  }
402  
403  int xen_pt_msix_update_remap(XenPCIPassthroughState *s, int bar_index)
404  {
405      XenPTMSIXEntry *entry;
406      int i, ret;
407  
408      if (!(s->msix && s->msix->bar_index == bar_index)) {
409          return 0;
410      }
411  
412      for (i = 0; i < s->msix->total_entries; i++) {
413          entry = &s->msix->msix_entry[i];
414          if (entry->pirq != XEN_PT_UNASSIGNED_PIRQ) {
415              ret = xc_domain_unbind_pt_irq(xen_xc, xen_domid, entry->pirq,
416                                            PT_IRQ_TYPE_MSI, 0, 0, 0, 0);
417              if (ret) {
418                  XEN_PT_ERR(&s->dev, "unbind MSI-X entry %d failed (err: %d)\n",
419                             entry->pirq, errno);
420              }
421              entry->updated = true;
422          }
423      }
424      return xen_pt_msix_update(s);
425  }
426  
427  static uint32_t get_entry_value(XenPTMSIXEntry *e, int offset)
428  {
429      assert(!(offset % sizeof(*e->latch)));
430      return e->latch[offset / sizeof(*e->latch)];
431  }
432  
433  static void set_entry_value(XenPTMSIXEntry *e, int offset, uint32_t val)
434  {
435      assert(!(offset % sizeof(*e->latch)));
436      e->latch[offset / sizeof(*e->latch)] = val;
437  }
438  
439  static void pci_msix_write(void *opaque, hwaddr addr,
440                             uint64_t val, unsigned size)
441  {
442      XenPCIPassthroughState *s = opaque;
443      XenPTMSIX *msix = s->msix;
444      XenPTMSIXEntry *entry;
445      unsigned int entry_nr, offset;
446  
447      entry_nr = addr / PCI_MSIX_ENTRY_SIZE;
448      if (entry_nr >= msix->total_entries) {
449          return;
450      }
451      entry = &msix->msix_entry[entry_nr];
452      offset = addr % PCI_MSIX_ENTRY_SIZE;
453  
454      if (offset != PCI_MSIX_ENTRY_VECTOR_CTRL) {
455          if (get_entry_value(entry, offset) == val
456              && entry->pirq != XEN_PT_UNASSIGNED_PIRQ) {
457              return;
458          }
459  
460          entry->updated = true;
461      } else if (msix->enabled && entry->updated &&
462                 !(val & PCI_MSIX_ENTRY_CTRL_MASKBIT)) {
463          const volatile uint32_t *vec_ctrl;
464  
465          /*
466           * If Xen intercepts the mask bit access, entry->vec_ctrl may not be
467           * up-to-date. Read from hardware directly.
468           */
469          vec_ctrl = s->msix->phys_iomem_base + entry_nr * PCI_MSIX_ENTRY_SIZE
470              + PCI_MSIX_ENTRY_VECTOR_CTRL;
471          xen_pt_msix_update_one(s, entry_nr, *vec_ctrl);
472      }
473  
474      set_entry_value(entry, offset, val);
475  }
476  
477  static uint64_t pci_msix_read(void *opaque, hwaddr addr,
478                                unsigned size)
479  {
480      XenPCIPassthroughState *s = opaque;
481      XenPTMSIX *msix = s->msix;
482      int entry_nr, offset;
483  
484      entry_nr = addr / PCI_MSIX_ENTRY_SIZE;
485      if (entry_nr < 0) {
486          XEN_PT_ERR(&s->dev, "asked MSI-X entry '%i' invalid!\n", entry_nr);
487          return 0;
488      }
489  
490      offset = addr % PCI_MSIX_ENTRY_SIZE;
491  
492      if (addr < msix->total_entries * PCI_MSIX_ENTRY_SIZE) {
493          return get_entry_value(&msix->msix_entry[entry_nr], offset);
494      } else {
495          /* Pending Bit Array (PBA) */
496          return *(uint32_t *)(msix->phys_iomem_base + addr);
497      }
498  }
499  
500  static bool pci_msix_accepts(void *opaque, hwaddr addr,
501                               unsigned size, bool is_write)
502  {
503      return !(addr & (size - 1));
504  }
505  
506  static const MemoryRegionOps pci_msix_ops = {
507      .read = pci_msix_read,
508      .write = pci_msix_write,
509      .endianness = DEVICE_NATIVE_ENDIAN,
510      .valid = {
511          .min_access_size = 4,
512          .max_access_size = 4,
513          .unaligned = false,
514          .accepts = pci_msix_accepts
515      },
516      .impl = {
517          .min_access_size = 4,
518          .max_access_size = 4,
519          .unaligned = false
520      }
521  };
522  
523  int xen_pt_msix_init(XenPCIPassthroughState *s, uint32_t base)
524  {
525      uint8_t id = 0;
526      uint16_t control = 0;
527      uint32_t table_off = 0;
528      int i, total_entries, bar_index;
529      XenHostPCIDevice *hd = &s->real_device;
530      PCIDevice *d = &s->dev;
531      int fd = -1;
532      XenPTMSIX *msix = NULL;
533      int rc = 0;
534  
535      rc = xen_host_pci_get_byte(hd, base + PCI_CAP_LIST_ID, &id);
536      if (rc) {
537          return rc;
538      }
539  
540      if (id != PCI_CAP_ID_MSIX) {
541          XEN_PT_ERR(d, "Invalid id %#x base %#x\n", id, base);
542          return -1;
543      }
544  
545      rc = xen_host_pci_get_word(hd, base + PCI_MSIX_FLAGS, &control);
546      if (rc) {
547          XEN_PT_ERR(d, "Failed to read PCI_MSIX_FLAGS field\n");
548          return rc;
549      }
550      total_entries = control & PCI_MSIX_FLAGS_QSIZE;
551      total_entries += 1;
552  
553      s->msix = g_malloc0(sizeof (XenPTMSIX)
554                          + total_entries * sizeof (XenPTMSIXEntry));
555      msix = s->msix;
556  
557      msix->total_entries = total_entries;
558      for (i = 0; i < total_entries; i++) {
559          msix->msix_entry[i].pirq = XEN_PT_UNASSIGNED_PIRQ;
560      }
561  
562      memory_region_init_io(&msix->mmio, OBJECT(s), &pci_msix_ops,
563                            s, "xen-pci-pt-msix",
564                            (total_entries * PCI_MSIX_ENTRY_SIZE
565                             + XC_PAGE_SIZE - 1)
566                            & XC_PAGE_MASK);
567  
568      rc = xen_host_pci_get_long(hd, base + PCI_MSIX_TABLE, &table_off);
569      if (rc) {
570          XEN_PT_ERR(d, "Failed to read PCI_MSIX_TABLE field\n");
571          goto error_out;
572      }
573      bar_index = msix->bar_index = table_off & PCI_MSIX_FLAGS_BIRMASK;
574      table_off = table_off & ~PCI_MSIX_FLAGS_BIRMASK;
575      msix->table_base = s->real_device.io_regions[bar_index].base_addr;
576      XEN_PT_LOG(d, "get MSI-X table BAR base 0x%"PRIx64"\n", msix->table_base);
577  
578      fd = open("/dev/mem", O_RDWR);
579      if (fd == -1) {
580          rc = -errno;
581          XEN_PT_ERR(d, "Can't open /dev/mem: %s\n", strerror(errno));
582          goto error_out;
583      }
584      XEN_PT_LOG(d, "table_off = %#x, total_entries = %d\n",
585                 table_off, total_entries);
586      msix->table_offset_adjust = table_off & 0x0fff;
587      msix->phys_iomem_base =
588          mmap(NULL,
589               total_entries * PCI_MSIX_ENTRY_SIZE + msix->table_offset_adjust,
590               PROT_READ,
591               MAP_SHARED | MAP_LOCKED,
592               fd,
593               msix->table_base + table_off - msix->table_offset_adjust);
594      close(fd);
595      if (msix->phys_iomem_base == MAP_FAILED) {
596          rc = -errno;
597          XEN_PT_ERR(d, "Can't map physical MSI-X table: %s\n", strerror(errno));
598          goto error_out;
599      }
600      msix->phys_iomem_base = (char *)msix->phys_iomem_base
601          + msix->table_offset_adjust;
602  
603      XEN_PT_LOG(d, "mapping physical MSI-X table to %p\n",
604                 msix->phys_iomem_base);
605  
606      memory_region_add_subregion_overlap(&s->bar[bar_index], table_off,
607                                          &msix->mmio,
608                                          2); /* Priority: pci default + 1 */
609  
610      return 0;
611  
612  error_out:
613      g_free(s->msix);
614      s->msix = NULL;
615      return rc;
616  }
617  
618  void xen_pt_msix_unmap(XenPCIPassthroughState *s)
619  {
620      XenPTMSIX *msix = s->msix;
621  
622      if (!msix) {
623          return;
624      }
625  
626      /* unmap the MSI-X memory mapped register area */
627      if (msix->phys_iomem_base) {
628          XEN_PT_LOG(&s->dev, "unmapping physical MSI-X table from %p\n",
629                     msix->phys_iomem_base);
630          munmap(msix->phys_iomem_base, msix->total_entries * PCI_MSIX_ENTRY_SIZE
631                 + msix->table_offset_adjust);
632      }
633  
634      memory_region_del_subregion(&s->bar[msix->bar_index], &msix->mmio);
635  }
636  
637  void xen_pt_msix_delete(XenPCIPassthroughState *s)
638  {
639      XenPTMSIX *msix = s->msix;
640  
641      if (!msix) {
642          return;
643      }
644  
645      object_unparent(OBJECT(&msix->mmio));
646  
647      g_free(s->msix);
648      s->msix = NULL;
649  }
650