1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2020 Intel Corporation. All rights reserved. */ 3 #include <linux/io-64-nonatomic-lo-hi.h> 4 #include <linux/moduleparam.h> 5 #include <linux/module.h> 6 #include <linux/delay.h> 7 #include <linux/sizes.h> 8 #include <linux/mutex.h> 9 #include <linux/list.h> 10 #include <linux/pci.h> 11 #include <linux/aer.h> 12 #include <linux/io.h> 13 #include "cxlmem.h" 14 #include "cxlpci.h" 15 #include "cxl.h" 16 17 /** 18 * DOC: cxl pci 19 * 20 * This implements the PCI exclusive functionality for a CXL device as it is 21 * defined by the Compute Express Link specification. CXL devices may surface 22 * certain functionality even if it isn't CXL enabled. While this driver is 23 * focused around the PCI specific aspects of a CXL device, it binds to the 24 * specific CXL memory device class code, and therefore the implementation of 25 * cxl_pci is focused around CXL memory devices. 26 * 27 * The driver has several responsibilities, mainly: 28 * - Create the memX device and register on the CXL bus. 29 * - Enumerate device's register interface and map them. 30 * - Registers nvdimm bridge device with cxl_core. 31 * - Registers a CXL mailbox with cxl_core. 32 */ 33 34 #define cxl_doorbell_busy(cxlds) \ 35 (readl((cxlds)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \ 36 CXLDEV_MBOX_CTRL_DOORBELL) 37 38 /* CXL 2.0 - 8.2.8.4 */ 39 #define CXL_MAILBOX_TIMEOUT_MS (2 * HZ) 40 41 /* 42 * CXL 2.0 ECN "Add Mailbox Ready Time" defines a capability field to 43 * dictate how long to wait for the mailbox to become ready. The new 44 * field allows the device to tell software the amount of time to wait 45 * before mailbox ready. This field per the spec theoretically allows 46 * for up to 255 seconds. 255 seconds is unreasonably long, its longer 47 * than the maximum SATA port link recovery wait. Default to 60 seconds 48 * until someone builds a CXL device that needs more time in practice. 49 */ 50 static unsigned short mbox_ready_timeout = 60; 51 module_param(mbox_ready_timeout, ushort, 0644); 52 MODULE_PARM_DESC(mbox_ready_timeout, "seconds to wait for mailbox ready"); 53 54 static int cxl_pci_mbox_wait_for_doorbell(struct cxl_dev_state *cxlds) 55 { 56 const unsigned long start = jiffies; 57 unsigned long end = start; 58 59 while (cxl_doorbell_busy(cxlds)) { 60 end = jiffies; 61 62 if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) { 63 /* Check again in case preempted before timeout test */ 64 if (!cxl_doorbell_busy(cxlds)) 65 break; 66 return -ETIMEDOUT; 67 } 68 cpu_relax(); 69 } 70 71 dev_dbg(cxlds->dev, "Doorbell wait took %dms", 72 jiffies_to_msecs(end) - jiffies_to_msecs(start)); 73 return 0; 74 } 75 76 #define cxl_err(dev, status, msg) \ 77 dev_err_ratelimited(dev, msg ", device state %s%s\n", \ 78 status & CXLMDEV_DEV_FATAL ? " fatal" : "", \ 79 status & CXLMDEV_FW_HALT ? " firmware-halt" : "") 80 81 #define cxl_cmd_err(dev, cmd, status, msg) \ 82 dev_err_ratelimited(dev, msg " (opcode: %#x), device state %s%s\n", \ 83 (cmd)->opcode, \ 84 status & CXLMDEV_DEV_FATAL ? " fatal" : "", \ 85 status & CXLMDEV_FW_HALT ? " firmware-halt" : "") 86 87 /** 88 * __cxl_pci_mbox_send_cmd() - Execute a mailbox command 89 * @mds: The memory device driver data 90 * @mbox_cmd: Command to send to the memory device. 91 * 92 * Context: Any context. Expects mbox_mutex to be held. 93 * Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success. 94 * Caller should check the return code in @mbox_cmd to make sure it 95 * succeeded. 96 * 97 * This is a generic form of the CXL mailbox send command thus only using the 98 * registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory 99 * devices, and perhaps other types of CXL devices may have further information 100 * available upon error conditions. Driver facilities wishing to send mailbox 101 * commands should use the wrapper command. 102 * 103 * The CXL spec allows for up to two mailboxes. The intention is for the primary 104 * mailbox to be OS controlled and the secondary mailbox to be used by system 105 * firmware. This allows the OS and firmware to communicate with the device and 106 * not need to coordinate with each other. The driver only uses the primary 107 * mailbox. 108 */ 109 static int __cxl_pci_mbox_send_cmd(struct cxl_memdev_state *mds, 110 struct cxl_mbox_cmd *mbox_cmd) 111 { 112 struct cxl_dev_state *cxlds = &mds->cxlds; 113 void __iomem *payload = cxlds->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET; 114 struct device *dev = cxlds->dev; 115 u64 cmd_reg, status_reg; 116 size_t out_len; 117 int rc; 118 119 lockdep_assert_held(&mds->mbox_mutex); 120 121 /* 122 * Here are the steps from 8.2.8.4 of the CXL 2.0 spec. 123 * 1. Caller reads MB Control Register to verify doorbell is clear 124 * 2. Caller writes Command Register 125 * 3. Caller writes Command Payload Registers if input payload is non-empty 126 * 4. Caller writes MB Control Register to set doorbell 127 * 5. Caller either polls for doorbell to be clear or waits for interrupt if configured 128 * 6. Caller reads MB Status Register to fetch Return code 129 * 7. If command successful, Caller reads Command Register to get Payload Length 130 * 8. If output payload is non-empty, host reads Command Payload Registers 131 * 132 * Hardware is free to do whatever it wants before the doorbell is rung, 133 * and isn't allowed to change anything after it clears the doorbell. As 134 * such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can 135 * also happen in any order (though some orders might not make sense). 136 */ 137 138 /* #1 */ 139 if (cxl_doorbell_busy(cxlds)) { 140 u64 md_status = 141 readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); 142 143 cxl_cmd_err(cxlds->dev, mbox_cmd, md_status, 144 "mailbox queue busy"); 145 return -EBUSY; 146 } 147 148 cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK, 149 mbox_cmd->opcode); 150 if (mbox_cmd->size_in) { 151 if (WARN_ON(!mbox_cmd->payload_in)) 152 return -EINVAL; 153 154 cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, 155 mbox_cmd->size_in); 156 memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in); 157 } 158 159 /* #2, #3 */ 160 writeq(cmd_reg, cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 161 162 /* #4 */ 163 dev_dbg(dev, "Sending command: 0x%04x\n", mbox_cmd->opcode); 164 writel(CXLDEV_MBOX_CTRL_DOORBELL, 165 cxlds->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET); 166 167 /* #5 */ 168 rc = cxl_pci_mbox_wait_for_doorbell(cxlds); 169 if (rc == -ETIMEDOUT) { 170 u64 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); 171 172 cxl_cmd_err(cxlds->dev, mbox_cmd, md_status, "mailbox timeout"); 173 return rc; 174 } 175 176 /* #6 */ 177 status_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET); 178 mbox_cmd->return_code = 179 FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg); 180 181 if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS) { 182 dev_dbg(dev, "Mailbox operation had an error: %s\n", 183 cxl_mbox_cmd_rc2str(mbox_cmd)); 184 return 0; /* completed but caller must check return_code */ 185 } 186 187 /* #7 */ 188 cmd_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 189 out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg); 190 191 /* #8 */ 192 if (out_len && mbox_cmd->payload_out) { 193 /* 194 * Sanitize the copy. If hardware misbehaves, out_len per the 195 * spec can actually be greater than the max allowed size (21 196 * bits available but spec defined 1M max). The caller also may 197 * have requested less data than the hardware supplied even 198 * within spec. 199 */ 200 size_t n; 201 202 n = min3(mbox_cmd->size_out, mds->payload_size, out_len); 203 memcpy_fromio(mbox_cmd->payload_out, payload, n); 204 mbox_cmd->size_out = n; 205 } else { 206 mbox_cmd->size_out = 0; 207 } 208 209 return 0; 210 } 211 212 static int cxl_pci_mbox_send(struct cxl_memdev_state *mds, 213 struct cxl_mbox_cmd *cmd) 214 { 215 int rc; 216 217 mutex_lock_io(&mds->mbox_mutex); 218 rc = __cxl_pci_mbox_send_cmd(mds, cmd); 219 mutex_unlock(&mds->mbox_mutex); 220 221 return rc; 222 } 223 224 static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds) 225 { 226 struct cxl_dev_state *cxlds = &mds->cxlds; 227 const int cap = readl(cxlds->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET); 228 struct device *dev = cxlds->dev; 229 unsigned long timeout; 230 u64 md_status; 231 232 timeout = jiffies + mbox_ready_timeout * HZ; 233 do { 234 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); 235 if (md_status & CXLMDEV_MBOX_IF_READY) 236 break; 237 if (msleep_interruptible(100)) 238 break; 239 } while (!time_after(jiffies, timeout)); 240 241 if (!(md_status & CXLMDEV_MBOX_IF_READY)) { 242 cxl_err(dev, md_status, "timeout awaiting mailbox ready"); 243 return -ETIMEDOUT; 244 } 245 246 /* 247 * A command may be in flight from a previous driver instance, 248 * think kexec, do one doorbell wait so that 249 * __cxl_pci_mbox_send_cmd() can assume that it is the only 250 * source for future doorbell busy events. 251 */ 252 if (cxl_pci_mbox_wait_for_doorbell(cxlds) != 0) { 253 cxl_err(dev, md_status, "timeout awaiting mailbox idle"); 254 return -ETIMEDOUT; 255 } 256 257 mds->mbox_send = cxl_pci_mbox_send; 258 mds->payload_size = 259 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap); 260 261 /* 262 * CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register 263 * 264 * If the size is too small, mandatory commands will not work and so 265 * there's no point in going forward. If the size is too large, there's 266 * no harm is soft limiting it. 267 */ 268 mds->payload_size = min_t(size_t, mds->payload_size, SZ_1M); 269 if (mds->payload_size < 256) { 270 dev_err(dev, "Mailbox is too small (%zub)", 271 mds->payload_size); 272 return -ENXIO; 273 } 274 275 dev_dbg(dev, "Mailbox payload sized %zu", mds->payload_size); 276 277 return 0; 278 } 279 280 static int cxl_map_regblock(struct pci_dev *pdev, struct cxl_register_map *map) 281 { 282 struct device *dev = &pdev->dev; 283 284 map->base = ioremap(map->resource, map->max_size); 285 if (!map->base) { 286 dev_err(dev, "failed to map registers\n"); 287 return -ENOMEM; 288 } 289 290 dev_dbg(dev, "Mapped CXL Memory Device resource %pa\n", &map->resource); 291 return 0; 292 } 293 294 static void cxl_unmap_regblock(struct pci_dev *pdev, 295 struct cxl_register_map *map) 296 { 297 iounmap(map->base); 298 map->base = NULL; 299 } 300 301 static int cxl_probe_regs(struct pci_dev *pdev, struct cxl_register_map *map) 302 { 303 struct cxl_component_reg_map *comp_map; 304 struct cxl_device_reg_map *dev_map; 305 struct device *dev = &pdev->dev; 306 void __iomem *base = map->base; 307 308 switch (map->reg_type) { 309 case CXL_REGLOC_RBI_COMPONENT: 310 comp_map = &map->component_map; 311 cxl_probe_component_regs(dev, base, comp_map); 312 if (!comp_map->hdm_decoder.valid) { 313 dev_err(dev, "HDM decoder registers not found\n"); 314 return -ENXIO; 315 } 316 317 if (!comp_map->ras.valid) 318 dev_dbg(dev, "RAS registers not found\n"); 319 320 dev_dbg(dev, "Set up component registers\n"); 321 break; 322 case CXL_REGLOC_RBI_MEMDEV: 323 dev_map = &map->device_map; 324 cxl_probe_device_regs(dev, base, dev_map); 325 if (!dev_map->status.valid || !dev_map->mbox.valid || 326 !dev_map->memdev.valid) { 327 dev_err(dev, "registers not found: %s%s%s\n", 328 !dev_map->status.valid ? "status " : "", 329 !dev_map->mbox.valid ? "mbox " : "", 330 !dev_map->memdev.valid ? "memdev " : ""); 331 return -ENXIO; 332 } 333 334 dev_dbg(dev, "Probing device registers...\n"); 335 break; 336 default: 337 break; 338 } 339 340 return 0; 341 } 342 343 static int cxl_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type, 344 struct cxl_register_map *map) 345 { 346 int rc; 347 348 rc = cxl_find_regblock(pdev, type, map); 349 if (rc) 350 return rc; 351 352 rc = cxl_map_regblock(pdev, map); 353 if (rc) 354 return rc; 355 356 rc = cxl_probe_regs(pdev, map); 357 cxl_unmap_regblock(pdev, map); 358 359 return rc; 360 } 361 362 /* 363 * Assume that any RCIEP that emits the CXL memory expander class code 364 * is an RCD 365 */ 366 static bool is_cxl_restricted(struct pci_dev *pdev) 367 { 368 return pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_END; 369 } 370 371 static int cxl_pci_ras_unmask(struct pci_dev *pdev) 372 { 373 struct pci_host_bridge *host_bridge = pci_find_host_bridge(pdev->bus); 374 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); 375 void __iomem *addr; 376 u32 orig_val, val, mask; 377 u16 cap; 378 int rc; 379 380 if (!cxlds->regs.ras) { 381 dev_dbg(&pdev->dev, "No RAS registers.\n"); 382 return 0; 383 } 384 385 /* BIOS has CXL error control */ 386 if (!host_bridge->native_cxl_error) 387 return -ENXIO; 388 389 rc = pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &cap); 390 if (rc) 391 return rc; 392 393 if (cap & PCI_EXP_DEVCTL_URRE) { 394 addr = cxlds->regs.ras + CXL_RAS_UNCORRECTABLE_MASK_OFFSET; 395 orig_val = readl(addr); 396 397 mask = CXL_RAS_UNCORRECTABLE_MASK_MASK | 398 CXL_RAS_UNCORRECTABLE_MASK_F256B_MASK; 399 val = orig_val & ~mask; 400 writel(val, addr); 401 dev_dbg(&pdev->dev, 402 "Uncorrectable RAS Errors Mask: %#x -> %#x\n", 403 orig_val, val); 404 } 405 406 if (cap & PCI_EXP_DEVCTL_CERE) { 407 addr = cxlds->regs.ras + CXL_RAS_CORRECTABLE_MASK_OFFSET; 408 orig_val = readl(addr); 409 val = orig_val & ~CXL_RAS_CORRECTABLE_MASK_MASK; 410 writel(val, addr); 411 dev_dbg(&pdev->dev, "Correctable RAS Errors Mask: %#x -> %#x\n", 412 orig_val, val); 413 } 414 415 return 0; 416 } 417 418 static void free_event_buf(void *buf) 419 { 420 kvfree(buf); 421 } 422 423 /* 424 * There is a single buffer for reading event logs from the mailbox. All logs 425 * share this buffer protected by the mds->event_log_lock. 426 */ 427 static int cxl_mem_alloc_event_buf(struct cxl_memdev_state *mds) 428 { 429 struct cxl_get_event_payload *buf; 430 431 buf = kvmalloc(mds->payload_size, GFP_KERNEL); 432 if (!buf) 433 return -ENOMEM; 434 mds->event.buf = buf; 435 436 return devm_add_action_or_reset(mds->cxlds.dev, free_event_buf, buf); 437 } 438 439 static int cxl_alloc_irq_vectors(struct pci_dev *pdev) 440 { 441 int nvecs; 442 443 /* 444 * Per CXL 3.0 3.1.1 CXL.io Endpoint a function on a CXL device must 445 * not generate INTx messages if that function participates in 446 * CXL.cache or CXL.mem. 447 * 448 * Additionally pci_alloc_irq_vectors() handles calling 449 * pci_free_irq_vectors() automatically despite not being called 450 * pcim_*. See pci_setup_msi_context(). 451 */ 452 nvecs = pci_alloc_irq_vectors(pdev, 1, CXL_PCI_DEFAULT_MAX_VECTORS, 453 PCI_IRQ_MSIX | PCI_IRQ_MSI); 454 if (nvecs < 1) { 455 dev_dbg(&pdev->dev, "Failed to alloc irq vectors: %d\n", nvecs); 456 return -ENXIO; 457 } 458 return 0; 459 } 460 461 struct cxl_dev_id { 462 struct cxl_dev_state *cxlds; 463 }; 464 465 static irqreturn_t cxl_event_thread(int irq, void *id) 466 { 467 struct cxl_dev_id *dev_id = id; 468 struct cxl_dev_state *cxlds = dev_id->cxlds; 469 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 470 u32 status; 471 472 do { 473 /* 474 * CXL 3.0 8.2.8.3.1: The lower 32 bits are the status; 475 * ignore the reserved upper 32 bits 476 */ 477 status = readl(cxlds->regs.status + CXLDEV_DEV_EVENT_STATUS_OFFSET); 478 /* Ignore logs unknown to the driver */ 479 status &= CXLDEV_EVENT_STATUS_ALL; 480 if (!status) 481 break; 482 cxl_mem_get_event_records(mds, status); 483 cond_resched(); 484 } while (status); 485 486 return IRQ_HANDLED; 487 } 488 489 static int cxl_event_req_irq(struct cxl_dev_state *cxlds, u8 setting) 490 { 491 struct device *dev = cxlds->dev; 492 struct pci_dev *pdev = to_pci_dev(dev); 493 struct cxl_dev_id *dev_id; 494 int irq; 495 496 if (FIELD_GET(CXLDEV_EVENT_INT_MODE_MASK, setting) != CXL_INT_MSI_MSIX) 497 return -ENXIO; 498 499 /* dev_id must be globally unique and must contain the cxlds */ 500 dev_id = devm_kzalloc(dev, sizeof(*dev_id), GFP_KERNEL); 501 if (!dev_id) 502 return -ENOMEM; 503 dev_id->cxlds = cxlds; 504 505 irq = pci_irq_vector(pdev, 506 FIELD_GET(CXLDEV_EVENT_INT_MSGNUM_MASK, setting)); 507 if (irq < 0) 508 return irq; 509 510 return devm_request_threaded_irq(dev, irq, NULL, cxl_event_thread, 511 IRQF_SHARED | IRQF_ONESHOT, NULL, 512 dev_id); 513 } 514 515 static int cxl_event_get_int_policy(struct cxl_memdev_state *mds, 516 struct cxl_event_interrupt_policy *policy) 517 { 518 struct cxl_mbox_cmd mbox_cmd = { 519 .opcode = CXL_MBOX_OP_GET_EVT_INT_POLICY, 520 .payload_out = policy, 521 .size_out = sizeof(*policy), 522 }; 523 int rc; 524 525 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 526 if (rc < 0) 527 dev_err(mds->cxlds.dev, 528 "Failed to get event interrupt policy : %d", rc); 529 530 return rc; 531 } 532 533 static int cxl_event_config_msgnums(struct cxl_memdev_state *mds, 534 struct cxl_event_interrupt_policy *policy) 535 { 536 struct cxl_mbox_cmd mbox_cmd; 537 int rc; 538 539 *policy = (struct cxl_event_interrupt_policy) { 540 .info_settings = CXL_INT_MSI_MSIX, 541 .warn_settings = CXL_INT_MSI_MSIX, 542 .failure_settings = CXL_INT_MSI_MSIX, 543 .fatal_settings = CXL_INT_MSI_MSIX, 544 }; 545 546 mbox_cmd = (struct cxl_mbox_cmd) { 547 .opcode = CXL_MBOX_OP_SET_EVT_INT_POLICY, 548 .payload_in = policy, 549 .size_in = sizeof(*policy), 550 }; 551 552 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 553 if (rc < 0) { 554 dev_err(mds->cxlds.dev, "Failed to set event interrupt policy : %d", 555 rc); 556 return rc; 557 } 558 559 /* Retrieve final interrupt settings */ 560 return cxl_event_get_int_policy(mds, policy); 561 } 562 563 static int cxl_event_irqsetup(struct cxl_memdev_state *mds) 564 { 565 struct cxl_dev_state *cxlds = &mds->cxlds; 566 struct cxl_event_interrupt_policy policy; 567 int rc; 568 569 rc = cxl_event_config_msgnums(mds, &policy); 570 if (rc) 571 return rc; 572 573 rc = cxl_event_req_irq(cxlds, policy.info_settings); 574 if (rc) { 575 dev_err(cxlds->dev, "Failed to get interrupt for event Info log\n"); 576 return rc; 577 } 578 579 rc = cxl_event_req_irq(cxlds, policy.warn_settings); 580 if (rc) { 581 dev_err(cxlds->dev, "Failed to get interrupt for event Warn log\n"); 582 return rc; 583 } 584 585 rc = cxl_event_req_irq(cxlds, policy.failure_settings); 586 if (rc) { 587 dev_err(cxlds->dev, "Failed to get interrupt for event Failure log\n"); 588 return rc; 589 } 590 591 rc = cxl_event_req_irq(cxlds, policy.fatal_settings); 592 if (rc) { 593 dev_err(cxlds->dev, "Failed to get interrupt for event Fatal log\n"); 594 return rc; 595 } 596 597 return 0; 598 } 599 600 static bool cxl_event_int_is_fw(u8 setting) 601 { 602 u8 mode = FIELD_GET(CXLDEV_EVENT_INT_MODE_MASK, setting); 603 604 return mode == CXL_INT_FW; 605 } 606 607 static int cxl_event_config(struct pci_host_bridge *host_bridge, 608 struct cxl_memdev_state *mds) 609 { 610 struct cxl_event_interrupt_policy policy; 611 int rc; 612 613 /* 614 * When BIOS maintains CXL error reporting control, it will process 615 * event records. Only one agent can do so. 616 */ 617 if (!host_bridge->native_cxl_error) 618 return 0; 619 620 rc = cxl_mem_alloc_event_buf(mds); 621 if (rc) 622 return rc; 623 624 rc = cxl_event_get_int_policy(mds, &policy); 625 if (rc) 626 return rc; 627 628 if (cxl_event_int_is_fw(policy.info_settings) || 629 cxl_event_int_is_fw(policy.warn_settings) || 630 cxl_event_int_is_fw(policy.failure_settings) || 631 cxl_event_int_is_fw(policy.fatal_settings)) { 632 dev_err(mds->cxlds.dev, 633 "FW still in control of Event Logs despite _OSC settings\n"); 634 return -EBUSY; 635 } 636 637 rc = cxl_event_irqsetup(mds); 638 if (rc) 639 return rc; 640 641 cxl_mem_get_event_records(mds, CXLDEV_EVENT_STATUS_ALL); 642 643 return 0; 644 } 645 646 static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 647 { 648 struct pci_host_bridge *host_bridge = pci_find_host_bridge(pdev->bus); 649 struct cxl_memdev_state *mds; 650 struct cxl_dev_state *cxlds; 651 struct cxl_register_map map; 652 struct cxl_memdev *cxlmd; 653 int rc; 654 655 /* 656 * Double check the anonymous union trickery in struct cxl_regs 657 * FIXME switch to struct_group() 658 */ 659 BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) != 660 offsetof(struct cxl_regs, device_regs.memdev)); 661 662 rc = pcim_enable_device(pdev); 663 if (rc) 664 return rc; 665 pci_set_master(pdev); 666 667 mds = cxl_memdev_state_create(&pdev->dev); 668 if (IS_ERR(mds)) 669 return PTR_ERR(mds); 670 cxlds = &mds->cxlds; 671 pci_set_drvdata(pdev, cxlds); 672 673 cxlds->rcd = is_cxl_restricted(pdev); 674 cxlds->serial = pci_get_dsn(pdev); 675 cxlds->cxl_dvsec = pci_find_dvsec_capability( 676 pdev, PCI_DVSEC_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE); 677 if (!cxlds->cxl_dvsec) 678 dev_warn(&pdev->dev, 679 "Device DVSEC not present, skip CXL.mem init\n"); 680 681 rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map); 682 if (rc) 683 return rc; 684 685 rc = cxl_map_device_regs(&pdev->dev, &cxlds->regs.device_regs, &map); 686 if (rc) 687 return rc; 688 689 /* 690 * If the component registers can't be found, the cxl_pci driver may 691 * still be useful for management functions so don't return an error. 692 */ 693 cxlds->component_reg_phys = CXL_RESOURCE_NONE; 694 rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, &map); 695 if (rc) 696 dev_warn(&pdev->dev, "No component registers (%d)\n", rc); 697 698 cxlds->component_reg_phys = map.resource; 699 700 rc = cxl_map_component_regs(&pdev->dev, &cxlds->regs.component, 701 &map, BIT(CXL_CM_CAP_CAP_ID_RAS)); 702 if (rc) 703 dev_dbg(&pdev->dev, "Failed to map RAS capability.\n"); 704 705 rc = cxl_await_media_ready(cxlds); 706 if (rc == 0) 707 cxlds->media_ready = true; 708 else 709 dev_warn(&pdev->dev, "Media not active (%d)\n", rc); 710 711 rc = cxl_pci_setup_mailbox(mds); 712 if (rc) 713 return rc; 714 715 rc = cxl_enumerate_cmds(mds); 716 if (rc) 717 return rc; 718 719 rc = cxl_set_timestamp(mds); 720 if (rc) 721 return rc; 722 723 rc = cxl_poison_state_init(mds); 724 if (rc) 725 return rc; 726 727 rc = cxl_dev_state_identify(mds); 728 if (rc) 729 return rc; 730 731 rc = cxl_mem_create_range_info(mds); 732 if (rc) 733 return rc; 734 735 rc = cxl_alloc_irq_vectors(pdev); 736 if (rc) 737 return rc; 738 739 cxlmd = devm_cxl_add_memdev(cxlds); 740 if (IS_ERR(cxlmd)) 741 return PTR_ERR(cxlmd); 742 743 rc = cxl_event_config(host_bridge, mds); 744 if (rc) 745 return rc; 746 747 rc = cxl_pci_ras_unmask(pdev); 748 if (rc) 749 dev_dbg(&pdev->dev, "No RAS reporting unmasked\n"); 750 751 pci_save_state(pdev); 752 753 return rc; 754 } 755 756 static const struct pci_device_id cxl_mem_pci_tbl[] = { 757 /* PCI class code for CXL.mem Type-3 Devices */ 758 { PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)}, 759 { /* terminate list */ }, 760 }; 761 MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl); 762 763 static pci_ers_result_t cxl_slot_reset(struct pci_dev *pdev) 764 { 765 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); 766 struct cxl_memdev *cxlmd = cxlds->cxlmd; 767 struct device *dev = &cxlmd->dev; 768 769 dev_info(&pdev->dev, "%s: restart CXL.mem after slot reset\n", 770 dev_name(dev)); 771 pci_restore_state(pdev); 772 if (device_attach(dev) <= 0) 773 return PCI_ERS_RESULT_DISCONNECT; 774 return PCI_ERS_RESULT_RECOVERED; 775 } 776 777 static void cxl_error_resume(struct pci_dev *pdev) 778 { 779 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); 780 struct cxl_memdev *cxlmd = cxlds->cxlmd; 781 struct device *dev = &cxlmd->dev; 782 783 dev_info(&pdev->dev, "%s: error resume %s\n", dev_name(dev), 784 dev->driver ? "successful" : "failed"); 785 } 786 787 static const struct pci_error_handlers cxl_error_handlers = { 788 .error_detected = cxl_error_detected, 789 .slot_reset = cxl_slot_reset, 790 .resume = cxl_error_resume, 791 .cor_error_detected = cxl_cor_error_detected, 792 }; 793 794 static struct pci_driver cxl_pci_driver = { 795 .name = KBUILD_MODNAME, 796 .id_table = cxl_mem_pci_tbl, 797 .probe = cxl_pci_probe, 798 .err_handler = &cxl_error_handlers, 799 .driver = { 800 .probe_type = PROBE_PREFER_ASYNCHRONOUS, 801 }, 802 }; 803 804 MODULE_LICENSE("GPL v2"); 805 module_pci_driver(cxl_pci_driver); 806 MODULE_IMPORT_NS(CXL); 807