1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2020 Intel Corporation. All rights reserved. */ 3 #include <linux/io-64-nonatomic-lo-hi.h> 4 #include <linux/moduleparam.h> 5 #include <linux/module.h> 6 #include <linux/delay.h> 7 #include <linux/sizes.h> 8 #include <linux/mutex.h> 9 #include <linux/list.h> 10 #include <linux/pci.h> 11 #include <linux/aer.h> 12 #include <linux/io.h> 13 #include "cxlmem.h" 14 #include "cxlpci.h" 15 #include "cxl.h" 16 17 /** 18 * DOC: cxl pci 19 * 20 * This implements the PCI exclusive functionality for a CXL device as it is 21 * defined by the Compute Express Link specification. CXL devices may surface 22 * certain functionality even if it isn't CXL enabled. While this driver is 23 * focused around the PCI specific aspects of a CXL device, it binds to the 24 * specific CXL memory device class code, and therefore the implementation of 25 * cxl_pci is focused around CXL memory devices. 26 * 27 * The driver has several responsibilities, mainly: 28 * - Create the memX device and register on the CXL bus. 29 * - Enumerate device's register interface and map them. 30 * - Registers nvdimm bridge device with cxl_core. 31 * - Registers a CXL mailbox with cxl_core. 32 */ 33 34 #define cxl_doorbell_busy(cxlds) \ 35 (readl((cxlds)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \ 36 CXLDEV_MBOX_CTRL_DOORBELL) 37 38 /* CXL 2.0 - 8.2.8.4 */ 39 #define CXL_MAILBOX_TIMEOUT_MS (2 * HZ) 40 41 /* 42 * CXL 2.0 ECN "Add Mailbox Ready Time" defines a capability field to 43 * dictate how long to wait for the mailbox to become ready. The new 44 * field allows the device to tell software the amount of time to wait 45 * before mailbox ready. This field per the spec theoretically allows 46 * for up to 255 seconds. 255 seconds is unreasonably long, its longer 47 * than the maximum SATA port link recovery wait. Default to 60 seconds 48 * until someone builds a CXL device that needs more time in practice. 49 */ 50 static unsigned short mbox_ready_timeout = 60; 51 module_param(mbox_ready_timeout, ushort, 0644); 52 MODULE_PARM_DESC(mbox_ready_timeout, "seconds to wait for mailbox ready"); 53 54 static int cxl_pci_mbox_wait_for_doorbell(struct cxl_dev_state *cxlds) 55 { 56 const unsigned long start = jiffies; 57 unsigned long end = start; 58 59 while (cxl_doorbell_busy(cxlds)) { 60 end = jiffies; 61 62 if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) { 63 /* Check again in case preempted before timeout test */ 64 if (!cxl_doorbell_busy(cxlds)) 65 break; 66 return -ETIMEDOUT; 67 } 68 cpu_relax(); 69 } 70 71 dev_dbg(cxlds->dev, "Doorbell wait took %dms", 72 jiffies_to_msecs(end) - jiffies_to_msecs(start)); 73 return 0; 74 } 75 76 #define cxl_err(dev, status, msg) \ 77 dev_err_ratelimited(dev, msg ", device state %s%s\n", \ 78 status & CXLMDEV_DEV_FATAL ? " fatal" : "", \ 79 status & CXLMDEV_FW_HALT ? " firmware-halt" : "") 80 81 #define cxl_cmd_err(dev, cmd, status, msg) \ 82 dev_err_ratelimited(dev, msg " (opcode: %#x), device state %s%s\n", \ 83 (cmd)->opcode, \ 84 status & CXLMDEV_DEV_FATAL ? " fatal" : "", \ 85 status & CXLMDEV_FW_HALT ? " firmware-halt" : "") 86 87 /** 88 * __cxl_pci_mbox_send_cmd() - Execute a mailbox command 89 * @mds: The memory device driver data 90 * @mbox_cmd: Command to send to the memory device. 91 * 92 * Context: Any context. Expects mbox_mutex to be held. 93 * Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success. 94 * Caller should check the return code in @mbox_cmd to make sure it 95 * succeeded. 96 * 97 * This is a generic form of the CXL mailbox send command thus only using the 98 * registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory 99 * devices, and perhaps other types of CXL devices may have further information 100 * available upon error conditions. Driver facilities wishing to send mailbox 101 * commands should use the wrapper command. 102 * 103 * The CXL spec allows for up to two mailboxes. The intention is for the primary 104 * mailbox to be OS controlled and the secondary mailbox to be used by system 105 * firmware. This allows the OS and firmware to communicate with the device and 106 * not need to coordinate with each other. The driver only uses the primary 107 * mailbox. 108 */ 109 static int __cxl_pci_mbox_send_cmd(struct cxl_memdev_state *mds, 110 struct cxl_mbox_cmd *mbox_cmd) 111 { 112 struct cxl_dev_state *cxlds = &mds->cxlds; 113 void __iomem *payload = cxlds->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET; 114 struct device *dev = cxlds->dev; 115 u64 cmd_reg, status_reg; 116 size_t out_len; 117 int rc; 118 119 lockdep_assert_held(&mds->mbox_mutex); 120 121 /* 122 * Here are the steps from 8.2.8.4 of the CXL 2.0 spec. 123 * 1. Caller reads MB Control Register to verify doorbell is clear 124 * 2. Caller writes Command Register 125 * 3. Caller writes Command Payload Registers if input payload is non-empty 126 * 4. Caller writes MB Control Register to set doorbell 127 * 5. Caller either polls for doorbell to be clear or waits for interrupt if configured 128 * 6. Caller reads MB Status Register to fetch Return code 129 * 7. If command successful, Caller reads Command Register to get Payload Length 130 * 8. If output payload is non-empty, host reads Command Payload Registers 131 * 132 * Hardware is free to do whatever it wants before the doorbell is rung, 133 * and isn't allowed to change anything after it clears the doorbell. As 134 * such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can 135 * also happen in any order (though some orders might not make sense). 136 */ 137 138 /* #1 */ 139 if (cxl_doorbell_busy(cxlds)) { 140 u64 md_status = 141 readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); 142 143 cxl_cmd_err(cxlds->dev, mbox_cmd, md_status, 144 "mailbox queue busy"); 145 return -EBUSY; 146 } 147 148 cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK, 149 mbox_cmd->opcode); 150 if (mbox_cmd->size_in) { 151 if (WARN_ON(!mbox_cmd->payload_in)) 152 return -EINVAL; 153 154 cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, 155 mbox_cmd->size_in); 156 memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in); 157 } 158 159 /* #2, #3 */ 160 writeq(cmd_reg, cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 161 162 /* #4 */ 163 dev_dbg(dev, "Sending command: 0x%04x\n", mbox_cmd->opcode); 164 writel(CXLDEV_MBOX_CTRL_DOORBELL, 165 cxlds->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET); 166 167 /* #5 */ 168 rc = cxl_pci_mbox_wait_for_doorbell(cxlds); 169 if (rc == -ETIMEDOUT) { 170 u64 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); 171 172 cxl_cmd_err(cxlds->dev, mbox_cmd, md_status, "mailbox timeout"); 173 return rc; 174 } 175 176 /* #6 */ 177 status_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET); 178 mbox_cmd->return_code = 179 FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg); 180 181 if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS) { 182 dev_dbg(dev, "Mailbox operation had an error: %s\n", 183 cxl_mbox_cmd_rc2str(mbox_cmd)); 184 return 0; /* completed but caller must check return_code */ 185 } 186 187 /* #7 */ 188 cmd_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 189 out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg); 190 191 /* #8 */ 192 if (out_len && mbox_cmd->payload_out) { 193 /* 194 * Sanitize the copy. If hardware misbehaves, out_len per the 195 * spec can actually be greater than the max allowed size (21 196 * bits available but spec defined 1M max). The caller also may 197 * have requested less data than the hardware supplied even 198 * within spec. 199 */ 200 size_t n; 201 202 n = min3(mbox_cmd->size_out, mds->payload_size, out_len); 203 memcpy_fromio(mbox_cmd->payload_out, payload, n); 204 mbox_cmd->size_out = n; 205 } else { 206 mbox_cmd->size_out = 0; 207 } 208 209 return 0; 210 } 211 212 static int cxl_pci_mbox_send(struct cxl_memdev_state *mds, 213 struct cxl_mbox_cmd *cmd) 214 { 215 int rc; 216 217 mutex_lock_io(&mds->mbox_mutex); 218 rc = __cxl_pci_mbox_send_cmd(mds, cmd); 219 mutex_unlock(&mds->mbox_mutex); 220 221 return rc; 222 } 223 224 static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds) 225 { 226 struct cxl_dev_state *cxlds = &mds->cxlds; 227 const int cap = readl(cxlds->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET); 228 struct device *dev = cxlds->dev; 229 unsigned long timeout; 230 u64 md_status; 231 232 timeout = jiffies + mbox_ready_timeout * HZ; 233 do { 234 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); 235 if (md_status & CXLMDEV_MBOX_IF_READY) 236 break; 237 if (msleep_interruptible(100)) 238 break; 239 } while (!time_after(jiffies, timeout)); 240 241 if (!(md_status & CXLMDEV_MBOX_IF_READY)) { 242 cxl_err(dev, md_status, "timeout awaiting mailbox ready"); 243 return -ETIMEDOUT; 244 } 245 246 /* 247 * A command may be in flight from a previous driver instance, 248 * think kexec, do one doorbell wait so that 249 * __cxl_pci_mbox_send_cmd() can assume that it is the only 250 * source for future doorbell busy events. 251 */ 252 if (cxl_pci_mbox_wait_for_doorbell(cxlds) != 0) { 253 cxl_err(dev, md_status, "timeout awaiting mailbox idle"); 254 return -ETIMEDOUT; 255 } 256 257 mds->mbox_send = cxl_pci_mbox_send; 258 mds->payload_size = 259 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap); 260 261 /* 262 * CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register 263 * 264 * If the size is too small, mandatory commands will not work and so 265 * there's no point in going forward. If the size is too large, there's 266 * no harm is soft limiting it. 267 */ 268 mds->payload_size = min_t(size_t, mds->payload_size, SZ_1M); 269 if (mds->payload_size < 256) { 270 dev_err(dev, "Mailbox is too small (%zub)", 271 mds->payload_size); 272 return -ENXIO; 273 } 274 275 dev_dbg(dev, "Mailbox payload sized %zu", mds->payload_size); 276 277 return 0; 278 } 279 280 static int cxl_map_regblock(struct pci_dev *pdev, struct cxl_register_map *map) 281 { 282 struct device *dev = &pdev->dev; 283 284 map->base = ioremap(map->resource, map->max_size); 285 if (!map->base) { 286 dev_err(dev, "failed to map registers\n"); 287 return -ENOMEM; 288 } 289 290 dev_dbg(dev, "Mapped CXL Memory Device resource %pa\n", &map->resource); 291 return 0; 292 } 293 294 static void cxl_unmap_regblock(struct pci_dev *pdev, 295 struct cxl_register_map *map) 296 { 297 iounmap(map->base); 298 map->base = NULL; 299 } 300 301 static int cxl_probe_regs(struct pci_dev *pdev, struct cxl_register_map *map) 302 { 303 struct cxl_component_reg_map *comp_map; 304 struct cxl_device_reg_map *dev_map; 305 struct device *dev = &pdev->dev; 306 void __iomem *base = map->base; 307 308 switch (map->reg_type) { 309 case CXL_REGLOC_RBI_COMPONENT: 310 comp_map = &map->component_map; 311 cxl_probe_component_regs(dev, base, comp_map); 312 if (!comp_map->hdm_decoder.valid) { 313 dev_err(dev, "HDM decoder registers not found\n"); 314 return -ENXIO; 315 } 316 317 if (!comp_map->ras.valid) 318 dev_dbg(dev, "RAS registers not found\n"); 319 320 dev_dbg(dev, "Set up component registers\n"); 321 break; 322 case CXL_REGLOC_RBI_MEMDEV: 323 dev_map = &map->device_map; 324 cxl_probe_device_regs(dev, base, dev_map); 325 if (!dev_map->status.valid || !dev_map->mbox.valid || 326 !dev_map->memdev.valid) { 327 dev_err(dev, "registers not found: %s%s%s\n", 328 !dev_map->status.valid ? "status " : "", 329 !dev_map->mbox.valid ? "mbox " : "", 330 !dev_map->memdev.valid ? "memdev " : ""); 331 return -ENXIO; 332 } 333 334 dev_dbg(dev, "Probing device registers...\n"); 335 break; 336 default: 337 break; 338 } 339 340 return 0; 341 } 342 343 static int cxl_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type, 344 struct cxl_register_map *map) 345 { 346 int rc; 347 348 rc = cxl_find_regblock(pdev, type, map); 349 if (rc) 350 return rc; 351 352 rc = cxl_map_regblock(pdev, map); 353 if (rc) 354 return rc; 355 356 rc = cxl_probe_regs(pdev, map); 357 cxl_unmap_regblock(pdev, map); 358 359 return rc; 360 } 361 362 /* 363 * Assume that any RCIEP that emits the CXL memory expander class code 364 * is an RCD 365 */ 366 static bool is_cxl_restricted(struct pci_dev *pdev) 367 { 368 return pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_END; 369 } 370 371 /* 372 * CXL v3.0 6.2.3 Table 6-4 373 * The table indicates that if PCIe Flit Mode is set, then CXL is in 256B flits 374 * mode, otherwise it's 68B flits mode. 375 */ 376 static bool cxl_pci_flit_256(struct pci_dev *pdev) 377 { 378 u16 lnksta2; 379 380 pcie_capability_read_word(pdev, PCI_EXP_LNKSTA2, &lnksta2); 381 return lnksta2 & PCI_EXP_LNKSTA2_FLIT; 382 } 383 384 static int cxl_pci_ras_unmask(struct pci_dev *pdev) 385 { 386 struct pci_host_bridge *host_bridge = pci_find_host_bridge(pdev->bus); 387 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); 388 void __iomem *addr; 389 u32 orig_val, val, mask; 390 u16 cap; 391 int rc; 392 393 if (!cxlds->regs.ras) { 394 dev_dbg(&pdev->dev, "No RAS registers.\n"); 395 return 0; 396 } 397 398 /* BIOS has CXL error control */ 399 if (!host_bridge->native_cxl_error) 400 return -ENXIO; 401 402 rc = pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &cap); 403 if (rc) 404 return rc; 405 406 if (cap & PCI_EXP_DEVCTL_URRE) { 407 addr = cxlds->regs.ras + CXL_RAS_UNCORRECTABLE_MASK_OFFSET; 408 orig_val = readl(addr); 409 410 mask = CXL_RAS_UNCORRECTABLE_MASK_MASK; 411 if (!cxl_pci_flit_256(pdev)) 412 mask &= ~CXL_RAS_UNCORRECTABLE_MASK_F256B_MASK; 413 val = orig_val & ~mask; 414 writel(val, addr); 415 dev_dbg(&pdev->dev, 416 "Uncorrectable RAS Errors Mask: %#x -> %#x\n", 417 orig_val, val); 418 } 419 420 if (cap & PCI_EXP_DEVCTL_CERE) { 421 addr = cxlds->regs.ras + CXL_RAS_CORRECTABLE_MASK_OFFSET; 422 orig_val = readl(addr); 423 val = orig_val & ~CXL_RAS_CORRECTABLE_MASK_MASK; 424 writel(val, addr); 425 dev_dbg(&pdev->dev, "Correctable RAS Errors Mask: %#x -> %#x\n", 426 orig_val, val); 427 } 428 429 return 0; 430 } 431 432 static void free_event_buf(void *buf) 433 { 434 kvfree(buf); 435 } 436 437 /* 438 * There is a single buffer for reading event logs from the mailbox. All logs 439 * share this buffer protected by the mds->event_log_lock. 440 */ 441 static int cxl_mem_alloc_event_buf(struct cxl_memdev_state *mds) 442 { 443 struct cxl_get_event_payload *buf; 444 445 buf = kvmalloc(mds->payload_size, GFP_KERNEL); 446 if (!buf) 447 return -ENOMEM; 448 mds->event.buf = buf; 449 450 return devm_add_action_or_reset(mds->cxlds.dev, free_event_buf, buf); 451 } 452 453 static int cxl_alloc_irq_vectors(struct pci_dev *pdev) 454 { 455 int nvecs; 456 457 /* 458 * Per CXL 3.0 3.1.1 CXL.io Endpoint a function on a CXL device must 459 * not generate INTx messages if that function participates in 460 * CXL.cache or CXL.mem. 461 * 462 * Additionally pci_alloc_irq_vectors() handles calling 463 * pci_free_irq_vectors() automatically despite not being called 464 * pcim_*. See pci_setup_msi_context(). 465 */ 466 nvecs = pci_alloc_irq_vectors(pdev, 1, CXL_PCI_DEFAULT_MAX_VECTORS, 467 PCI_IRQ_MSIX | PCI_IRQ_MSI); 468 if (nvecs < 1) { 469 dev_dbg(&pdev->dev, "Failed to alloc irq vectors: %d\n", nvecs); 470 return -ENXIO; 471 } 472 return 0; 473 } 474 475 struct cxl_dev_id { 476 struct cxl_dev_state *cxlds; 477 }; 478 479 static irqreturn_t cxl_event_thread(int irq, void *id) 480 { 481 struct cxl_dev_id *dev_id = id; 482 struct cxl_dev_state *cxlds = dev_id->cxlds; 483 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 484 u32 status; 485 486 do { 487 /* 488 * CXL 3.0 8.2.8.3.1: The lower 32 bits are the status; 489 * ignore the reserved upper 32 bits 490 */ 491 status = readl(cxlds->regs.status + CXLDEV_DEV_EVENT_STATUS_OFFSET); 492 /* Ignore logs unknown to the driver */ 493 status &= CXLDEV_EVENT_STATUS_ALL; 494 if (!status) 495 break; 496 cxl_mem_get_event_records(mds, status); 497 cond_resched(); 498 } while (status); 499 500 return IRQ_HANDLED; 501 } 502 503 static int cxl_event_req_irq(struct cxl_dev_state *cxlds, u8 setting) 504 { 505 struct device *dev = cxlds->dev; 506 struct pci_dev *pdev = to_pci_dev(dev); 507 struct cxl_dev_id *dev_id; 508 int irq; 509 510 if (FIELD_GET(CXLDEV_EVENT_INT_MODE_MASK, setting) != CXL_INT_MSI_MSIX) 511 return -ENXIO; 512 513 /* dev_id must be globally unique and must contain the cxlds */ 514 dev_id = devm_kzalloc(dev, sizeof(*dev_id), GFP_KERNEL); 515 if (!dev_id) 516 return -ENOMEM; 517 dev_id->cxlds = cxlds; 518 519 irq = pci_irq_vector(pdev, 520 FIELD_GET(CXLDEV_EVENT_INT_MSGNUM_MASK, setting)); 521 if (irq < 0) 522 return irq; 523 524 return devm_request_threaded_irq(dev, irq, NULL, cxl_event_thread, 525 IRQF_SHARED | IRQF_ONESHOT, NULL, 526 dev_id); 527 } 528 529 static int cxl_event_get_int_policy(struct cxl_memdev_state *mds, 530 struct cxl_event_interrupt_policy *policy) 531 { 532 struct cxl_mbox_cmd mbox_cmd = { 533 .opcode = CXL_MBOX_OP_GET_EVT_INT_POLICY, 534 .payload_out = policy, 535 .size_out = sizeof(*policy), 536 }; 537 int rc; 538 539 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 540 if (rc < 0) 541 dev_err(mds->cxlds.dev, 542 "Failed to get event interrupt policy : %d", rc); 543 544 return rc; 545 } 546 547 static int cxl_event_config_msgnums(struct cxl_memdev_state *mds, 548 struct cxl_event_interrupt_policy *policy) 549 { 550 struct cxl_mbox_cmd mbox_cmd; 551 int rc; 552 553 *policy = (struct cxl_event_interrupt_policy) { 554 .info_settings = CXL_INT_MSI_MSIX, 555 .warn_settings = CXL_INT_MSI_MSIX, 556 .failure_settings = CXL_INT_MSI_MSIX, 557 .fatal_settings = CXL_INT_MSI_MSIX, 558 }; 559 560 mbox_cmd = (struct cxl_mbox_cmd) { 561 .opcode = CXL_MBOX_OP_SET_EVT_INT_POLICY, 562 .payload_in = policy, 563 .size_in = sizeof(*policy), 564 }; 565 566 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 567 if (rc < 0) { 568 dev_err(mds->cxlds.dev, "Failed to set event interrupt policy : %d", 569 rc); 570 return rc; 571 } 572 573 /* Retrieve final interrupt settings */ 574 return cxl_event_get_int_policy(mds, policy); 575 } 576 577 static int cxl_event_irqsetup(struct cxl_memdev_state *mds) 578 { 579 struct cxl_dev_state *cxlds = &mds->cxlds; 580 struct cxl_event_interrupt_policy policy; 581 int rc; 582 583 rc = cxl_event_config_msgnums(mds, &policy); 584 if (rc) 585 return rc; 586 587 rc = cxl_event_req_irq(cxlds, policy.info_settings); 588 if (rc) { 589 dev_err(cxlds->dev, "Failed to get interrupt for event Info log\n"); 590 return rc; 591 } 592 593 rc = cxl_event_req_irq(cxlds, policy.warn_settings); 594 if (rc) { 595 dev_err(cxlds->dev, "Failed to get interrupt for event Warn log\n"); 596 return rc; 597 } 598 599 rc = cxl_event_req_irq(cxlds, policy.failure_settings); 600 if (rc) { 601 dev_err(cxlds->dev, "Failed to get interrupt for event Failure log\n"); 602 return rc; 603 } 604 605 rc = cxl_event_req_irq(cxlds, policy.fatal_settings); 606 if (rc) { 607 dev_err(cxlds->dev, "Failed to get interrupt for event Fatal log\n"); 608 return rc; 609 } 610 611 return 0; 612 } 613 614 static bool cxl_event_int_is_fw(u8 setting) 615 { 616 u8 mode = FIELD_GET(CXLDEV_EVENT_INT_MODE_MASK, setting); 617 618 return mode == CXL_INT_FW; 619 } 620 621 static int cxl_event_config(struct pci_host_bridge *host_bridge, 622 struct cxl_memdev_state *mds) 623 { 624 struct cxl_event_interrupt_policy policy; 625 int rc; 626 627 /* 628 * When BIOS maintains CXL error reporting control, it will process 629 * event records. Only one agent can do so. 630 */ 631 if (!host_bridge->native_cxl_error) 632 return 0; 633 634 rc = cxl_mem_alloc_event_buf(mds); 635 if (rc) 636 return rc; 637 638 rc = cxl_event_get_int_policy(mds, &policy); 639 if (rc) 640 return rc; 641 642 if (cxl_event_int_is_fw(policy.info_settings) || 643 cxl_event_int_is_fw(policy.warn_settings) || 644 cxl_event_int_is_fw(policy.failure_settings) || 645 cxl_event_int_is_fw(policy.fatal_settings)) { 646 dev_err(mds->cxlds.dev, 647 "FW still in control of Event Logs despite _OSC settings\n"); 648 return -EBUSY; 649 } 650 651 rc = cxl_event_irqsetup(mds); 652 if (rc) 653 return rc; 654 655 cxl_mem_get_event_records(mds, CXLDEV_EVENT_STATUS_ALL); 656 657 return 0; 658 } 659 660 static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 661 { 662 struct pci_host_bridge *host_bridge = pci_find_host_bridge(pdev->bus); 663 struct cxl_memdev_state *mds; 664 struct cxl_dev_state *cxlds; 665 struct cxl_register_map map; 666 struct cxl_memdev *cxlmd; 667 int rc; 668 669 /* 670 * Double check the anonymous union trickery in struct cxl_regs 671 * FIXME switch to struct_group() 672 */ 673 BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) != 674 offsetof(struct cxl_regs, device_regs.memdev)); 675 676 rc = pcim_enable_device(pdev); 677 if (rc) 678 return rc; 679 pci_set_master(pdev); 680 681 mds = cxl_memdev_state_create(&pdev->dev); 682 if (IS_ERR(mds)) 683 return PTR_ERR(mds); 684 cxlds = &mds->cxlds; 685 pci_set_drvdata(pdev, cxlds); 686 687 cxlds->rcd = is_cxl_restricted(pdev); 688 cxlds->serial = pci_get_dsn(pdev); 689 cxlds->cxl_dvsec = pci_find_dvsec_capability( 690 pdev, PCI_DVSEC_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE); 691 if (!cxlds->cxl_dvsec) 692 dev_warn(&pdev->dev, 693 "Device DVSEC not present, skip CXL.mem init\n"); 694 695 rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map); 696 if (rc) 697 return rc; 698 699 rc = cxl_map_device_regs(&pdev->dev, &cxlds->regs.device_regs, &map); 700 if (rc) 701 return rc; 702 703 /* 704 * If the component registers can't be found, the cxl_pci driver may 705 * still be useful for management functions so don't return an error. 706 */ 707 cxlds->component_reg_phys = CXL_RESOURCE_NONE; 708 rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, &map); 709 if (rc) 710 dev_warn(&pdev->dev, "No component registers (%d)\n", rc); 711 712 cxlds->component_reg_phys = map.resource; 713 714 rc = cxl_map_component_regs(&pdev->dev, &cxlds->regs.component, 715 &map, BIT(CXL_CM_CAP_CAP_ID_RAS)); 716 if (rc) 717 dev_dbg(&pdev->dev, "Failed to map RAS capability.\n"); 718 719 rc = cxl_await_media_ready(cxlds); 720 if (rc == 0) 721 cxlds->media_ready = true; 722 else 723 dev_warn(&pdev->dev, "Media not active (%d)\n", rc); 724 725 rc = cxl_pci_setup_mailbox(mds); 726 if (rc) 727 return rc; 728 729 rc = cxl_enumerate_cmds(mds); 730 if (rc) 731 return rc; 732 733 rc = cxl_set_timestamp(mds); 734 if (rc) 735 return rc; 736 737 rc = cxl_poison_state_init(mds); 738 if (rc) 739 return rc; 740 741 rc = cxl_dev_state_identify(mds); 742 if (rc) 743 return rc; 744 745 rc = cxl_mem_create_range_info(mds); 746 if (rc) 747 return rc; 748 749 rc = cxl_alloc_irq_vectors(pdev); 750 if (rc) 751 return rc; 752 753 cxlmd = devm_cxl_add_memdev(cxlds); 754 if (IS_ERR(cxlmd)) 755 return PTR_ERR(cxlmd); 756 757 rc = cxl_event_config(host_bridge, mds); 758 if (rc) 759 return rc; 760 761 rc = cxl_pci_ras_unmask(pdev); 762 if (rc) 763 dev_dbg(&pdev->dev, "No RAS reporting unmasked\n"); 764 765 pci_save_state(pdev); 766 767 return rc; 768 } 769 770 static const struct pci_device_id cxl_mem_pci_tbl[] = { 771 /* PCI class code for CXL.mem Type-3 Devices */ 772 { PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)}, 773 { /* terminate list */ }, 774 }; 775 MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl); 776 777 static pci_ers_result_t cxl_slot_reset(struct pci_dev *pdev) 778 { 779 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); 780 struct cxl_memdev *cxlmd = cxlds->cxlmd; 781 struct device *dev = &cxlmd->dev; 782 783 dev_info(&pdev->dev, "%s: restart CXL.mem after slot reset\n", 784 dev_name(dev)); 785 pci_restore_state(pdev); 786 if (device_attach(dev) <= 0) 787 return PCI_ERS_RESULT_DISCONNECT; 788 return PCI_ERS_RESULT_RECOVERED; 789 } 790 791 static void cxl_error_resume(struct pci_dev *pdev) 792 { 793 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); 794 struct cxl_memdev *cxlmd = cxlds->cxlmd; 795 struct device *dev = &cxlmd->dev; 796 797 dev_info(&pdev->dev, "%s: error resume %s\n", dev_name(dev), 798 dev->driver ? "successful" : "failed"); 799 } 800 801 static const struct pci_error_handlers cxl_error_handlers = { 802 .error_detected = cxl_error_detected, 803 .slot_reset = cxl_slot_reset, 804 .resume = cxl_error_resume, 805 .cor_error_detected = cxl_cor_error_detected, 806 }; 807 808 static struct pci_driver cxl_pci_driver = { 809 .name = KBUILD_MODNAME, 810 .id_table = cxl_mem_pci_tbl, 811 .probe = cxl_pci_probe, 812 .err_handler = &cxl_error_handlers, 813 .driver = { 814 .probe_type = PROBE_PREFER_ASYNCHRONOUS, 815 }, 816 }; 817 818 MODULE_LICENSE("GPL v2"); 819 module_pci_driver(cxl_pci_driver); 820 MODULE_IMPORT_NS(CXL); 821