1 /* 2 * PCI Expander Bridge Device Emulation 3 * 4 * Copyright (C) 2015 Red Hat Inc 5 * 6 * Authors: 7 * Marcel Apfelbaum <marcel@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2 or later. 10 * See the COPYING file in the top-level directory. 11 */ 12 13 #include "qemu/osdep.h" 14 #include "qapi/error.h" 15 #include "hw/pci/pci.h" 16 #include "hw/pci/pci_bus.h" 17 #include "hw/pci/pci_host.h" 18 #include "hw/qdev-properties.h" 19 #include "hw/pci/pci_bridge.h" 20 #include "hw/cxl/cxl.h" 21 #include "qemu/range.h" 22 #include "qemu/error-report.h" 23 #include "qemu/module.h" 24 #include "sysemu/numa.h" 25 #include "hw/boards.h" 26 #include "qom/object.h" 27 28 enum BusType { PCI, PCIE, CXL }; 29 30 #define TYPE_PXB_BUS "pxb-bus" 31 typedef struct PXBBus PXBBus; 32 DECLARE_INSTANCE_CHECKER(PXBBus, PXB_BUS, 33 TYPE_PXB_BUS) 34 35 #define TYPE_PXB_PCIE_BUS "pxb-pcie-bus" 36 DECLARE_INSTANCE_CHECKER(PXBBus, PXB_PCIE_BUS, 37 TYPE_PXB_PCIE_BUS) 38 39 #define TYPE_PXB_CXL_BUS "pxb-cxl-bus" 40 DECLARE_INSTANCE_CHECKER(PXBBus, PXB_CXL_BUS, 41 TYPE_PXB_CXL_BUS) 42 43 struct PXBBus { 44 /*< private >*/ 45 PCIBus parent_obj; 46 /*< public >*/ 47 48 char bus_path[8]; 49 }; 50 51 #define TYPE_PXB_DEVICE "pxb" 52 typedef struct PXBDev PXBDev; 53 DECLARE_INSTANCE_CHECKER(PXBDev, PXB_DEV, 54 TYPE_PXB_DEVICE) 55 56 #define TYPE_PXB_PCIE_DEVICE "pxb-pcie" 57 DECLARE_INSTANCE_CHECKER(PXBDev, PXB_PCIE_DEV, 58 TYPE_PXB_PCIE_DEVICE) 59 60 static PXBDev *convert_to_pxb(PCIDevice *dev) 61 { 62 /* A CXL PXB's parent bus is PCIe, so the normal check won't work */ 63 if (object_dynamic_cast(OBJECT(dev), TYPE_PXB_CXL_DEVICE)) { 64 return PXB_CXL_DEV(dev); 65 } 66 67 return pci_bus_is_express(pci_get_bus(dev)) 68 ? PXB_PCIE_DEV(dev) : PXB_DEV(dev); 69 } 70 71 static GList *pxb_dev_list; 72 73 #define TYPE_PXB_HOST "pxb-host" 74 75 static int pxb_bus_num(PCIBus *bus) 76 { 77 PXBDev *pxb = convert_to_pxb(bus->parent_dev); 78 79 return pxb->bus_nr; 80 } 81 82 static uint16_t pxb_bus_numa_node(PCIBus *bus) 83 { 84 PXBDev *pxb = convert_to_pxb(bus->parent_dev); 85 86 return pxb->numa_node; 87 } 88 89 static void pxb_bus_class_init(ObjectClass *class, void *data) 90 { 91 PCIBusClass *pbc = PCI_BUS_CLASS(class); 92 93 pbc->bus_num = pxb_bus_num; 94 pbc->numa_node = pxb_bus_numa_node; 95 } 96 97 static const TypeInfo pxb_bus_info = { 98 .name = TYPE_PXB_BUS, 99 .parent = TYPE_PCI_BUS, 100 .instance_size = sizeof(PXBBus), 101 .class_init = pxb_bus_class_init, 102 }; 103 104 static const TypeInfo pxb_pcie_bus_info = { 105 .name = TYPE_PXB_PCIE_BUS, 106 .parent = TYPE_PCIE_BUS, 107 .instance_size = sizeof(PXBBus), 108 .class_init = pxb_bus_class_init, 109 }; 110 111 static const TypeInfo pxb_cxl_bus_info = { 112 .name = TYPE_PXB_CXL_BUS, 113 .parent = TYPE_CXL_BUS, 114 .instance_size = sizeof(PXBBus), 115 .class_init = pxb_bus_class_init, 116 }; 117 118 static const char *pxb_host_root_bus_path(PCIHostState *host_bridge, 119 PCIBus *rootbus) 120 { 121 PXBBus *bus = pci_bus_is_cxl(rootbus) ? 122 PXB_CXL_BUS(rootbus) : 123 pci_bus_is_express(rootbus) ? PXB_PCIE_BUS(rootbus) : 124 PXB_BUS(rootbus); 125 126 snprintf(bus->bus_path, 8, "0000:%02x", pxb_bus_num(rootbus)); 127 return bus->bus_path; 128 } 129 130 static char *pxb_host_ofw_unit_address(const SysBusDevice *dev) 131 { 132 const PCIHostState *pxb_host; 133 const PCIBus *pxb_bus; 134 const PXBDev *pxb_dev; 135 int position; 136 const DeviceState *pxb_dev_base; 137 const PCIHostState *main_host; 138 const SysBusDevice *main_host_sbd; 139 140 pxb_host = PCI_HOST_BRIDGE(dev); 141 pxb_bus = pxb_host->bus; 142 pxb_dev = convert_to_pxb(pxb_bus->parent_dev); 143 position = g_list_index(pxb_dev_list, pxb_dev); 144 assert(position >= 0); 145 146 pxb_dev_base = DEVICE(pxb_dev); 147 main_host = PCI_HOST_BRIDGE(pxb_dev_base->parent_bus->parent); 148 main_host_sbd = SYS_BUS_DEVICE(main_host); 149 150 if (main_host_sbd->num_mmio > 0) { 151 return g_strdup_printf(TARGET_FMT_plx ",%x", 152 main_host_sbd->mmio[0].addr, position + 1); 153 } 154 if (main_host_sbd->num_pio > 0) { 155 return g_strdup_printf("i%04x,%x", 156 main_host_sbd->pio[0], position + 1); 157 } 158 return NULL; 159 } 160 161 static void pxb_host_class_init(ObjectClass *class, void *data) 162 { 163 DeviceClass *dc = DEVICE_CLASS(class); 164 SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(class); 165 PCIHostBridgeClass *hc = PCI_HOST_BRIDGE_CLASS(class); 166 167 dc->fw_name = "pci"; 168 /* Reason: Internal part of the pxb/pxb-pcie device, not usable by itself */ 169 dc->user_creatable = false; 170 sbc->explicit_ofw_unit_address = pxb_host_ofw_unit_address; 171 hc->root_bus_path = pxb_host_root_bus_path; 172 } 173 174 static const TypeInfo pxb_host_info = { 175 .name = TYPE_PXB_HOST, 176 .parent = TYPE_PCI_HOST_BRIDGE, 177 .class_init = pxb_host_class_init, 178 }; 179 180 static void pxb_cxl_realize(DeviceState *dev, Error **errp) 181 { 182 MachineState *ms = MACHINE(qdev_get_machine()); 183 SysBusDevice *sbd = SYS_BUS_DEVICE(dev); 184 CXLHost *cxl = PXB_CXL_HOST(dev); 185 CXLComponentState *cxl_cstate = &cxl->cxl_cstate; 186 struct MemoryRegion *mr = &cxl_cstate->crb.component_registers; 187 hwaddr offset; 188 189 cxl_component_register_block_init(OBJECT(dev), cxl_cstate, 190 TYPE_PXB_CXL_HOST); 191 sysbus_init_mmio(sbd, mr); 192 193 offset = memory_region_size(mr) * ms->cxl_devices_state->next_mr_idx; 194 if (offset > memory_region_size(&ms->cxl_devices_state->host_mr)) { 195 error_setg(errp, "Insufficient space for pxb cxl host register space"); 196 return; 197 } 198 199 memory_region_add_subregion(&ms->cxl_devices_state->host_mr, offset, mr); 200 ms->cxl_devices_state->next_mr_idx++; 201 } 202 203 static void pxb_cxl_host_class_init(ObjectClass *class, void *data) 204 { 205 DeviceClass *dc = DEVICE_CLASS(class); 206 PCIHostBridgeClass *hc = PCI_HOST_BRIDGE_CLASS(class); 207 208 hc->root_bus_path = pxb_host_root_bus_path; 209 dc->fw_name = "cxl"; 210 dc->realize = pxb_cxl_realize; 211 /* Reason: Internal part of the pxb/pxb-pcie device, not usable by itself */ 212 dc->user_creatable = false; 213 } 214 215 /* 216 * This is a device to handle the MMIO for a CXL host bridge. It does nothing 217 * else. 218 */ 219 static const TypeInfo cxl_host_info = { 220 .name = TYPE_PXB_CXL_HOST, 221 .parent = TYPE_PCI_HOST_BRIDGE, 222 .instance_size = sizeof(CXLHost), 223 .class_init = pxb_cxl_host_class_init, 224 }; 225 226 /* 227 * Registers the PXB bus as a child of pci host root bus. 228 */ 229 static void pxb_register_bus(PCIDevice *dev, PCIBus *pxb_bus, Error **errp) 230 { 231 PCIBus *bus = pci_get_bus(dev); 232 int pxb_bus_num = pci_bus_num(pxb_bus); 233 234 if (bus->parent_dev) { 235 error_setg(errp, "PXB devices can be attached only to root bus"); 236 return; 237 } 238 239 QLIST_FOREACH(bus, &bus->child, sibling) { 240 if (pci_bus_num(bus) == pxb_bus_num) { 241 error_setg(errp, "Bus %d is already in use", pxb_bus_num); 242 return; 243 } 244 } 245 QLIST_INSERT_HEAD(&pci_get_bus(dev)->child, pxb_bus, sibling); 246 } 247 248 static int pxb_map_irq_fn(PCIDevice *pci_dev, int pin) 249 { 250 PCIDevice *pxb = pci_get_bus(pci_dev)->parent_dev; 251 252 /* 253 * First carry out normal swizzle to handle 254 * multple root ports on a pxb instance. 255 */ 256 pin = pci_swizzle_map_irq_fn(pci_dev, pin); 257 258 /* 259 * The bios does not index the pxb slot number when 260 * it computes the IRQ because it resides on bus 0 261 * and not on the current bus. 262 * However QEMU routes the irq through bus 0 and adds 263 * the pxb slot to the IRQ computation of the PXB 264 * device. 265 * 266 * Synchronize between bios and QEMU by canceling 267 * pxb's effect. 268 */ 269 return pin - PCI_SLOT(pxb->devfn); 270 } 271 272 static void pxb_dev_reset(DeviceState *dev) 273 { 274 CXLHost *cxl = PXB_CXL_DEV(dev)->cxl.cxl_host_bridge; 275 CXLComponentState *cxl_cstate = &cxl->cxl_cstate; 276 uint32_t *reg_state = cxl_cstate->crb.cache_mem_registers; 277 uint32_t *write_msk = cxl_cstate->crb.cache_mem_regs_write_mask; 278 279 cxl_component_register_init_common(reg_state, write_msk, CXL2_ROOT_PORT); 280 ARRAY_FIELD_DP32(reg_state, CXL_HDM_DECODER_CAPABILITY, TARGET_COUNT, 8); 281 } 282 283 static gint pxb_compare(gconstpointer a, gconstpointer b) 284 { 285 const PXBDev *pxb_a = a, *pxb_b = b; 286 287 return pxb_a->bus_nr < pxb_b->bus_nr ? -1 : 288 pxb_a->bus_nr > pxb_b->bus_nr ? 1 : 289 0; 290 } 291 292 static void pxb_dev_realize_common(PCIDevice *dev, enum BusType type, 293 Error **errp) 294 { 295 PXBDev *pxb = convert_to_pxb(dev); 296 DeviceState *ds, *bds = NULL; 297 PCIBus *bus; 298 const char *dev_name = NULL; 299 Error *local_err = NULL; 300 MachineState *ms = MACHINE(qdev_get_machine()); 301 302 if (ms->numa_state == NULL) { 303 error_setg(errp, "NUMA is not supported by this machine-type"); 304 return; 305 } 306 307 if (pxb->numa_node != NUMA_NODE_UNASSIGNED && 308 pxb->numa_node >= ms->numa_state->num_nodes) { 309 error_setg(errp, "Illegal numa node %d", pxb->numa_node); 310 return; 311 } 312 313 if (dev->qdev.id && *dev->qdev.id) { 314 dev_name = dev->qdev.id; 315 } 316 317 ds = qdev_new(type == CXL ? TYPE_PXB_CXL_HOST : TYPE_PXB_HOST); 318 if (type == PCIE) { 319 bus = pci_root_bus_new(ds, dev_name, NULL, NULL, 0, TYPE_PXB_PCIE_BUS); 320 } else if (type == CXL) { 321 bus = pci_root_bus_new(ds, dev_name, NULL, NULL, 0, TYPE_PXB_CXL_BUS); 322 bus->flags |= PCI_BUS_CXL; 323 PXB_CXL_DEV(dev)->cxl.cxl_host_bridge = PXB_CXL_HOST(ds); 324 } else { 325 bus = pci_root_bus_new(ds, "pxb-internal", NULL, NULL, 0, TYPE_PXB_BUS); 326 bds = qdev_new("pci-bridge"); 327 bds->id = g_strdup(dev_name); 328 qdev_prop_set_uint8(bds, PCI_BRIDGE_DEV_PROP_CHASSIS_NR, pxb->bus_nr); 329 qdev_prop_set_bit(bds, PCI_BRIDGE_DEV_PROP_SHPC, false); 330 } 331 332 bus->parent_dev = dev; 333 bus->address_space_mem = pci_get_bus(dev)->address_space_mem; 334 bus->address_space_io = pci_get_bus(dev)->address_space_io; 335 bus->map_irq = pxb_map_irq_fn; 336 337 PCI_HOST_BRIDGE(ds)->bus = bus; 338 PCI_HOST_BRIDGE(ds)->bypass_iommu = pxb->bypass_iommu; 339 340 pxb_register_bus(dev, bus, &local_err); 341 if (local_err) { 342 error_propagate(errp, local_err); 343 goto err_register_bus; 344 } 345 346 sysbus_realize_and_unref(SYS_BUS_DEVICE(ds), &error_fatal); 347 if (bds) { 348 qdev_realize_and_unref(bds, &bus->qbus, &error_fatal); 349 } 350 351 pci_word_test_and_set_mask(dev->config + PCI_STATUS, 352 PCI_STATUS_66MHZ | PCI_STATUS_FAST_BACK); 353 pci_config_set_class(dev->config, PCI_CLASS_BRIDGE_HOST); 354 355 pxb_dev_list = g_list_insert_sorted(pxb_dev_list, pxb, pxb_compare); 356 return; 357 358 err_register_bus: 359 object_unref(OBJECT(bds)); 360 object_unparent(OBJECT(bus)); 361 object_unref(OBJECT(ds)); 362 } 363 364 static void pxb_dev_realize(PCIDevice *dev, Error **errp) 365 { 366 if (pci_bus_is_express(pci_get_bus(dev))) { 367 error_setg(errp, "pxb devices cannot reside on a PCIe bus"); 368 return; 369 } 370 371 pxb_dev_realize_common(dev, PCI, errp); 372 } 373 374 static void pxb_dev_exitfn(PCIDevice *pci_dev) 375 { 376 PXBDev *pxb = convert_to_pxb(pci_dev); 377 378 pxb_dev_list = g_list_remove(pxb_dev_list, pxb); 379 } 380 381 static Property pxb_dev_properties[] = { 382 /* Note: 0 is not a legal PXB bus number. */ 383 DEFINE_PROP_UINT8("bus_nr", PXBDev, bus_nr, 0), 384 DEFINE_PROP_UINT16("numa_node", PXBDev, numa_node, NUMA_NODE_UNASSIGNED), 385 DEFINE_PROP_BOOL("bypass_iommu", PXBDev, bypass_iommu, false), 386 DEFINE_PROP_END_OF_LIST(), 387 }; 388 389 static void pxb_dev_class_init(ObjectClass *klass, void *data) 390 { 391 DeviceClass *dc = DEVICE_CLASS(klass); 392 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); 393 394 k->realize = pxb_dev_realize; 395 k->exit = pxb_dev_exitfn; 396 k->vendor_id = PCI_VENDOR_ID_REDHAT; 397 k->device_id = PCI_DEVICE_ID_REDHAT_PXB; 398 k->class_id = PCI_CLASS_BRIDGE_HOST; 399 400 dc->desc = "PCI Expander Bridge"; 401 device_class_set_props(dc, pxb_dev_properties); 402 dc->hotpluggable = false; 403 set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories); 404 } 405 406 static const TypeInfo pxb_dev_info = { 407 .name = TYPE_PXB_DEVICE, 408 .parent = TYPE_PCI_DEVICE, 409 .instance_size = sizeof(PXBDev), 410 .class_init = pxb_dev_class_init, 411 .interfaces = (InterfaceInfo[]) { 412 { INTERFACE_CONVENTIONAL_PCI_DEVICE }, 413 { }, 414 }, 415 }; 416 417 static void pxb_pcie_dev_realize(PCIDevice *dev, Error **errp) 418 { 419 if (!pci_bus_is_express(pci_get_bus(dev))) { 420 error_setg(errp, "pxb-pcie devices cannot reside on a PCI bus"); 421 return; 422 } 423 424 pxb_dev_realize_common(dev, PCIE, errp); 425 } 426 427 static void pxb_pcie_dev_class_init(ObjectClass *klass, void *data) 428 { 429 DeviceClass *dc = DEVICE_CLASS(klass); 430 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); 431 432 k->realize = pxb_pcie_dev_realize; 433 k->exit = pxb_dev_exitfn; 434 k->vendor_id = PCI_VENDOR_ID_REDHAT; 435 k->device_id = PCI_DEVICE_ID_REDHAT_PXB_PCIE; 436 k->class_id = PCI_CLASS_BRIDGE_HOST; 437 438 dc->desc = "PCI Express Expander Bridge"; 439 device_class_set_props(dc, pxb_dev_properties); 440 dc->hotpluggable = false; 441 set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories); 442 } 443 444 static const TypeInfo pxb_pcie_dev_info = { 445 .name = TYPE_PXB_PCIE_DEVICE, 446 .parent = TYPE_PCI_DEVICE, 447 .instance_size = sizeof(PXBDev), 448 .class_init = pxb_pcie_dev_class_init, 449 .interfaces = (InterfaceInfo[]) { 450 { INTERFACE_CONVENTIONAL_PCI_DEVICE }, 451 { }, 452 }, 453 }; 454 455 static void pxb_cxl_dev_realize(PCIDevice *dev, Error **errp) 456 { 457 MachineState *ms = MACHINE(qdev_get_machine()); 458 459 /* A CXL PXB's parent bus is still PCIe */ 460 if (!pci_bus_is_express(pci_get_bus(dev))) { 461 error_setg(errp, "pxb-cxl devices cannot reside on a PCI bus"); 462 return; 463 } 464 if (!ms->cxl_devices_state->is_enabled) { 465 error_setg(errp, "Machine does not have cxl=on"); 466 return; 467 } 468 469 pxb_dev_realize_common(dev, CXL, errp); 470 pxb_dev_reset(DEVICE(dev)); 471 } 472 473 static void pxb_cxl_dev_class_init(ObjectClass *klass, void *data) 474 { 475 DeviceClass *dc = DEVICE_CLASS(klass); 476 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); 477 478 k->realize = pxb_cxl_dev_realize; 479 k->exit = pxb_dev_exitfn; 480 /* 481 * XXX: These types of bridges don't actually show up in the hierarchy so 482 * vendor, device, class, etc. ids are intentionally left out. 483 */ 484 485 dc->desc = "CXL Host Bridge"; 486 device_class_set_props(dc, pxb_dev_properties); 487 set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories); 488 489 /* Host bridges aren't hotpluggable. FIXME: spec reference */ 490 dc->hotpluggable = false; 491 dc->reset = pxb_dev_reset; 492 } 493 494 static const TypeInfo pxb_cxl_dev_info = { 495 .name = TYPE_PXB_CXL_DEVICE, 496 .parent = TYPE_PCI_DEVICE, 497 .instance_size = sizeof(PXBDev), 498 .class_init = pxb_cxl_dev_class_init, 499 .interfaces = 500 (InterfaceInfo[]){ 501 { INTERFACE_CONVENTIONAL_PCI_DEVICE }, 502 {}, 503 }, 504 }; 505 506 static void pxb_register_types(void) 507 { 508 type_register_static(&pxb_bus_info); 509 type_register_static(&pxb_pcie_bus_info); 510 type_register_static(&pxb_cxl_bus_info); 511 type_register_static(&pxb_host_info); 512 type_register_static(&cxl_host_info); 513 type_register_static(&pxb_dev_info); 514 type_register_static(&pxb_pcie_dev_info); 515 type_register_static(&pxb_cxl_dev_info); 516 } 517 518 type_init(pxb_register_types) 519