1 /* 2 * PCI Backend Operations - respond to PCI requests from Frontend 3 * 4 * Author: Ryan Wilson <hap9@epoch.ncsc.mil> 5 */ 6 7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 9 #include <linux/module.h> 10 #include <linux/wait.h> 11 #include <linux/bitops.h> 12 #include <xen/events.h> 13 #include <linux/sched.h> 14 #include "pciback.h" 15 16 int verbose_request; 17 module_param(verbose_request, int, 0644); 18 19 static irqreturn_t xen_pcibk_guest_interrupt(int irq, void *dev_id); 20 21 /* Ensure a device is has the fake IRQ handler "turned on/off" and is 22 * ready to be exported. This MUST be run after xen_pcibk_reset_device 23 * which does the actual PCI device enable/disable. 24 */ 25 static void xen_pcibk_control_isr(struct pci_dev *dev, int reset) 26 { 27 struct xen_pcibk_dev_data *dev_data; 28 int rc; 29 int enable = 0; 30 31 dev_data = pci_get_drvdata(dev); 32 if (!dev_data) 33 return; 34 35 /* We don't deal with bridges */ 36 if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL) 37 return; 38 39 if (reset) { 40 dev_data->enable_intx = 0; 41 dev_data->ack_intr = 0; 42 } 43 enable = dev_data->enable_intx; 44 45 /* Asked to disable, but ISR isn't runnig */ 46 if (!enable && !dev_data->isr_on) 47 return; 48 49 /* Squirrel away the IRQs in the dev_data. We need this 50 * b/c when device transitions to MSI, the dev->irq is 51 * overwritten with the MSI vector. 52 */ 53 if (enable) 54 dev_data->irq = dev->irq; 55 56 /* 57 * SR-IOV devices in all use MSI-X and have no legacy 58 * interrupts, so inhibit creating a fake IRQ handler for them. 59 */ 60 if (dev_data->irq == 0) 61 goto out; 62 63 dev_dbg(&dev->dev, "%s: #%d %s %s%s %s-> %s\n", 64 dev_data->irq_name, 65 dev_data->irq, 66 pci_is_enabled(dev) ? "on" : "off", 67 dev->msi_enabled ? "MSI" : "", 68 dev->msix_enabled ? "MSI/X" : "", 69 dev_data->isr_on ? "enable" : "disable", 70 enable ? "enable" : "disable"); 71 72 if (enable) { 73 rc = request_irq(dev_data->irq, 74 xen_pcibk_guest_interrupt, IRQF_SHARED, 75 dev_data->irq_name, dev); 76 if (rc) { 77 dev_err(&dev->dev, "%s: failed to install fake IRQ " \ 78 "handler for IRQ %d! (rc:%d)\n", 79 dev_data->irq_name, dev_data->irq, rc); 80 goto out; 81 } 82 } else { 83 free_irq(dev_data->irq, dev); 84 dev_data->irq = 0; 85 } 86 dev_data->isr_on = enable; 87 dev_data->ack_intr = enable; 88 out: 89 dev_dbg(&dev->dev, "%s: #%d %s %s%s %s\n", 90 dev_data->irq_name, 91 dev_data->irq, 92 pci_is_enabled(dev) ? "on" : "off", 93 dev->msi_enabled ? "MSI" : "", 94 dev->msix_enabled ? "MSI/X" : "", 95 enable ? (dev_data->isr_on ? "enabled" : "failed to enable") : 96 (dev_data->isr_on ? "failed to disable" : "disabled")); 97 } 98 99 /* Ensure a device is "turned off" and ready to be exported. 100 * (Also see xen_pcibk_config_reset to ensure virtual configuration space is 101 * ready to be re-exported) 102 */ 103 void xen_pcibk_reset_device(struct pci_dev *dev) 104 { 105 u16 cmd; 106 107 xen_pcibk_control_isr(dev, 1 /* reset device */); 108 109 /* Disable devices (but not bridges) */ 110 if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL) { 111 #ifdef CONFIG_PCI_MSI 112 /* The guest could have been abruptly killed without 113 * disabling MSI/MSI-X interrupts.*/ 114 if (dev->msix_enabled) 115 pci_disable_msix(dev); 116 if (dev->msi_enabled) 117 pci_disable_msi(dev); 118 #endif 119 if (pci_is_enabled(dev)) 120 pci_disable_device(dev); 121 122 pci_write_config_word(dev, PCI_COMMAND, 0); 123 124 dev->is_busmaster = 0; 125 } else { 126 pci_read_config_word(dev, PCI_COMMAND, &cmd); 127 if (cmd & (PCI_COMMAND_INVALIDATE)) { 128 cmd &= ~(PCI_COMMAND_INVALIDATE); 129 pci_write_config_word(dev, PCI_COMMAND, cmd); 130 131 dev->is_busmaster = 0; 132 } 133 } 134 } 135 136 #ifdef CONFIG_PCI_MSI 137 static 138 int xen_pcibk_enable_msi(struct xen_pcibk_device *pdev, 139 struct pci_dev *dev, struct xen_pci_op *op) 140 { 141 struct xen_pcibk_dev_data *dev_data; 142 int status; 143 144 if (unlikely(verbose_request)) 145 printk(KERN_DEBUG DRV_NAME ": %s: enable MSI\n", pci_name(dev)); 146 147 status = pci_enable_msi(dev); 148 149 if (status) { 150 pr_warn_ratelimited("%s: error enabling MSI for guest %u: err %d\n", 151 pci_name(dev), pdev->xdev->otherend_id, 152 status); 153 op->value = 0; 154 return XEN_PCI_ERR_op_failed; 155 } 156 157 /* The value the guest needs is actually the IDT vector, not the 158 * the local domain's IRQ number. */ 159 160 op->value = dev->irq ? xen_pirq_from_irq(dev->irq) : 0; 161 if (unlikely(verbose_request)) 162 printk(KERN_DEBUG DRV_NAME ": %s: MSI: %d\n", pci_name(dev), 163 op->value); 164 165 dev_data = pci_get_drvdata(dev); 166 if (dev_data) 167 dev_data->ack_intr = 0; 168 169 return 0; 170 } 171 172 static 173 int xen_pcibk_disable_msi(struct xen_pcibk_device *pdev, 174 struct pci_dev *dev, struct xen_pci_op *op) 175 { 176 struct xen_pcibk_dev_data *dev_data; 177 178 if (unlikely(verbose_request)) 179 printk(KERN_DEBUG DRV_NAME ": %s: disable MSI\n", 180 pci_name(dev)); 181 pci_disable_msi(dev); 182 183 op->value = dev->irq ? xen_pirq_from_irq(dev->irq) : 0; 184 if (unlikely(verbose_request)) 185 printk(KERN_DEBUG DRV_NAME ": %s: MSI: %d\n", pci_name(dev), 186 op->value); 187 dev_data = pci_get_drvdata(dev); 188 if (dev_data) 189 dev_data->ack_intr = 1; 190 return 0; 191 } 192 193 static 194 int xen_pcibk_enable_msix(struct xen_pcibk_device *pdev, 195 struct pci_dev *dev, struct xen_pci_op *op) 196 { 197 struct xen_pcibk_dev_data *dev_data; 198 int i, result; 199 struct msix_entry *entries; 200 201 if (unlikely(verbose_request)) 202 printk(KERN_DEBUG DRV_NAME ": %s: enable MSI-X\n", 203 pci_name(dev)); 204 if (op->value > SH_INFO_MAX_VEC) 205 return -EINVAL; 206 207 entries = kmalloc(op->value * sizeof(*entries), GFP_KERNEL); 208 if (entries == NULL) 209 return -ENOMEM; 210 211 for (i = 0; i < op->value; i++) { 212 entries[i].entry = op->msix_entries[i].entry; 213 entries[i].vector = op->msix_entries[i].vector; 214 } 215 216 result = pci_enable_msix_exact(dev, entries, op->value); 217 if (result == 0) { 218 for (i = 0; i < op->value; i++) { 219 op->msix_entries[i].entry = entries[i].entry; 220 if (entries[i].vector) 221 op->msix_entries[i].vector = 222 xen_pirq_from_irq(entries[i].vector); 223 if (unlikely(verbose_request)) 224 printk(KERN_DEBUG DRV_NAME ": %s: " \ 225 "MSI-X[%d]: %d\n", 226 pci_name(dev), i, 227 op->msix_entries[i].vector); 228 } 229 } else 230 pr_warn_ratelimited("%s: error enabling MSI-X for guest %u: err %d!\n", 231 pci_name(dev), pdev->xdev->otherend_id, 232 result); 233 kfree(entries); 234 235 op->value = result; 236 dev_data = pci_get_drvdata(dev); 237 if (dev_data) 238 dev_data->ack_intr = 0; 239 240 return result > 0 ? 0 : result; 241 } 242 243 static 244 int xen_pcibk_disable_msix(struct xen_pcibk_device *pdev, 245 struct pci_dev *dev, struct xen_pci_op *op) 246 { 247 struct xen_pcibk_dev_data *dev_data; 248 if (unlikely(verbose_request)) 249 printk(KERN_DEBUG DRV_NAME ": %s: disable MSI-X\n", 250 pci_name(dev)); 251 pci_disable_msix(dev); 252 253 /* 254 * SR-IOV devices (which don't have any legacy IRQ) have 255 * an undefined IRQ value of zero. 256 */ 257 op->value = dev->irq ? xen_pirq_from_irq(dev->irq) : 0; 258 if (unlikely(verbose_request)) 259 printk(KERN_DEBUG DRV_NAME ": %s: MSI-X: %d\n", pci_name(dev), 260 op->value); 261 dev_data = pci_get_drvdata(dev); 262 if (dev_data) 263 dev_data->ack_intr = 1; 264 return 0; 265 } 266 #endif 267 /* 268 * Now the same evtchn is used for both pcifront conf_read_write request 269 * as well as pcie aer front end ack. We use a new work_queue to schedule 270 * xen_pcibk conf_read_write service for avoiding confict with aer_core 271 * do_recovery job which also use the system default work_queue 272 */ 273 void xen_pcibk_test_and_schedule_op(struct xen_pcibk_device *pdev) 274 { 275 /* Check that frontend is requesting an operation and that we are not 276 * already processing a request */ 277 if (test_bit(_XEN_PCIF_active, (unsigned long *)&pdev->sh_info->flags) 278 && !test_and_set_bit(_PDEVF_op_active, &pdev->flags)) { 279 queue_work(xen_pcibk_wq, &pdev->op_work); 280 } 281 /*_XEN_PCIB_active should have been cleared by pcifront. And also make 282 sure xen_pcibk is waiting for ack by checking _PCIB_op_pending*/ 283 if (!test_bit(_XEN_PCIB_active, (unsigned long *)&pdev->sh_info->flags) 284 && test_bit(_PCIB_op_pending, &pdev->flags)) { 285 wake_up(&xen_pcibk_aer_wait_queue); 286 } 287 } 288 289 /* Performing the configuration space reads/writes must not be done in atomic 290 * context because some of the pci_* functions can sleep (mostly due to ACPI 291 * use of semaphores). This function is intended to be called from a work 292 * queue in process context taking a struct xen_pcibk_device as a parameter */ 293 294 void xen_pcibk_do_op(struct work_struct *data) 295 { 296 struct xen_pcibk_device *pdev = 297 container_of(data, struct xen_pcibk_device, op_work); 298 struct pci_dev *dev; 299 struct xen_pcibk_dev_data *dev_data = NULL; 300 struct xen_pci_op *op = &pdev->sh_info->op; 301 int test_intx = 0; 302 303 dev = xen_pcibk_get_pci_dev(pdev, op->domain, op->bus, op->devfn); 304 305 if (dev == NULL) 306 op->err = XEN_PCI_ERR_dev_not_found; 307 else { 308 dev_data = pci_get_drvdata(dev); 309 if (dev_data) 310 test_intx = dev_data->enable_intx; 311 switch (op->cmd) { 312 case XEN_PCI_OP_conf_read: 313 op->err = xen_pcibk_config_read(dev, 314 op->offset, op->size, &op->value); 315 break; 316 case XEN_PCI_OP_conf_write: 317 op->err = xen_pcibk_config_write(dev, 318 op->offset, op->size, op->value); 319 break; 320 #ifdef CONFIG_PCI_MSI 321 case XEN_PCI_OP_enable_msi: 322 op->err = xen_pcibk_enable_msi(pdev, dev, op); 323 break; 324 case XEN_PCI_OP_disable_msi: 325 op->err = xen_pcibk_disable_msi(pdev, dev, op); 326 break; 327 case XEN_PCI_OP_enable_msix: 328 op->err = xen_pcibk_enable_msix(pdev, dev, op); 329 break; 330 case XEN_PCI_OP_disable_msix: 331 op->err = xen_pcibk_disable_msix(pdev, dev, op); 332 break; 333 #endif 334 default: 335 op->err = XEN_PCI_ERR_not_implemented; 336 break; 337 } 338 } 339 if (!op->err && dev && dev_data) { 340 /* Transition detected */ 341 if ((dev_data->enable_intx != test_intx)) 342 xen_pcibk_control_isr(dev, 0 /* no reset */); 343 } 344 /* Tell the driver domain that we're done. */ 345 wmb(); 346 clear_bit(_XEN_PCIF_active, (unsigned long *)&pdev->sh_info->flags); 347 notify_remote_via_irq(pdev->evtchn_irq); 348 349 /* Mark that we're done. */ 350 smp_mb__before_clear_bit(); /* /after/ clearing PCIF_active */ 351 clear_bit(_PDEVF_op_active, &pdev->flags); 352 smp_mb__after_clear_bit(); /* /before/ final check for work */ 353 354 /* Check to see if the driver domain tried to start another request in 355 * between clearing _XEN_PCIF_active and clearing _PDEVF_op_active. 356 */ 357 xen_pcibk_test_and_schedule_op(pdev); 358 } 359 360 irqreturn_t xen_pcibk_handle_event(int irq, void *dev_id) 361 { 362 struct xen_pcibk_device *pdev = dev_id; 363 364 xen_pcibk_test_and_schedule_op(pdev); 365 366 return IRQ_HANDLED; 367 } 368 static irqreturn_t xen_pcibk_guest_interrupt(int irq, void *dev_id) 369 { 370 struct pci_dev *dev = (struct pci_dev *)dev_id; 371 struct xen_pcibk_dev_data *dev_data = pci_get_drvdata(dev); 372 373 if (dev_data->isr_on && dev_data->ack_intr) { 374 dev_data->handled++; 375 if ((dev_data->handled % 1000) == 0) { 376 if (xen_test_irq_shared(irq)) { 377 pr_info("%s IRQ line is not shared " 378 "with other domains. Turning ISR off\n", 379 dev_data->irq_name); 380 dev_data->ack_intr = 0; 381 } 382 } 383 return IRQ_HANDLED; 384 } 385 return IRQ_NONE; 386 } 387