121e9f767SBen Widawsky // SPDX-License-Identifier: GPL-2.0-only 221e9f767SBen Widawsky /* Copyright(c) 2020 Intel Corporation. All rights reserved. */ 34faf31b4SDan Williams #include <linux/io-64-nonatomic-lo-hi.h> 4229e8828SBen Widawsky #include <linux/moduleparam.h> 521e9f767SBen Widawsky #include <linux/module.h> 6229e8828SBen Widawsky #include <linux/delay.h> 721e9f767SBen Widawsky #include <linux/sizes.h> 821e9f767SBen Widawsky #include <linux/mutex.h> 930af9729SIra Weiny #include <linux/list.h> 1021e9f767SBen Widawsky #include <linux/pci.h> 113eddcc93SIra Weiny #include <linux/pci-doe.h> 1221e9f767SBen Widawsky #include <linux/io.h> 135161a55cSBen Widawsky #include "cxlmem.h" 14af9cae9fSDan Williams #include "cxlpci.h" 1521e9f767SBen Widawsky #include "cxl.h" 1621e9f767SBen Widawsky 1721e9f767SBen Widawsky /** 1821e9f767SBen Widawsky * DOC: cxl pci 1921e9f767SBen Widawsky * 2021e9f767SBen Widawsky * This implements the PCI exclusive functionality for a CXL device as it is 2121e9f767SBen Widawsky * defined by the Compute Express Link specification. CXL devices may surface 22ed97afb5SBen Widawsky * certain functionality even if it isn't CXL enabled. While this driver is 23ed97afb5SBen Widawsky * focused around the PCI specific aspects of a CXL device, it binds to the 24ed97afb5SBen Widawsky * specific CXL memory device class code, and therefore the implementation of 25ed97afb5SBen Widawsky * cxl_pci is focused around CXL memory devices. 2621e9f767SBen Widawsky * 2721e9f767SBen Widawsky * The driver has several responsibilities, mainly: 2821e9f767SBen Widawsky * - Create the memX device and register on the CXL bus. 2921e9f767SBen Widawsky * - Enumerate device's register interface and map them. 30ed97afb5SBen Widawsky * - Registers nvdimm bridge device with cxl_core. 31ed97afb5SBen Widawsky * - Registers a CXL mailbox with cxl_core. 3221e9f767SBen Widawsky */ 3321e9f767SBen Widawsky 345e2411aeSIra Weiny #define cxl_doorbell_busy(cxlds) \ 355e2411aeSIra Weiny (readl((cxlds)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \ 3621e9f767SBen Widawsky CXLDEV_MBOX_CTRL_DOORBELL) 3721e9f767SBen Widawsky 3821e9f767SBen Widawsky /* CXL 2.0 - 8.2.8.4 */ 3921e9f767SBen Widawsky #define CXL_MAILBOX_TIMEOUT_MS (2 * HZ) 4021e9f767SBen Widawsky 41229e8828SBen Widawsky /* 42229e8828SBen Widawsky * CXL 2.0 ECN "Add Mailbox Ready Time" defines a capability field to 43229e8828SBen Widawsky * dictate how long to wait for the mailbox to become ready. The new 44229e8828SBen Widawsky * field allows the device to tell software the amount of time to wait 45229e8828SBen Widawsky * before mailbox ready. This field per the spec theoretically allows 46229e8828SBen Widawsky * for up to 255 seconds. 255 seconds is unreasonably long, its longer 47229e8828SBen Widawsky * than the maximum SATA port link recovery wait. Default to 60 seconds 48229e8828SBen Widawsky * until someone builds a CXL device that needs more time in practice. 49229e8828SBen Widawsky */ 50229e8828SBen Widawsky static unsigned short mbox_ready_timeout = 60; 51229e8828SBen Widawsky module_param(mbox_ready_timeout, ushort, 0644); 522e4ba0ecSDan Williams MODULE_PARM_DESC(mbox_ready_timeout, "seconds to wait for mailbox ready"); 53229e8828SBen Widawsky 545e2411aeSIra Weiny static int cxl_pci_mbox_wait_for_doorbell(struct cxl_dev_state *cxlds) 5521e9f767SBen Widawsky { 5621e9f767SBen Widawsky const unsigned long start = jiffies; 5721e9f767SBen Widawsky unsigned long end = start; 5821e9f767SBen Widawsky 595e2411aeSIra Weiny while (cxl_doorbell_busy(cxlds)) { 6021e9f767SBen Widawsky end = jiffies; 6121e9f767SBen Widawsky 6221e9f767SBen Widawsky if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) { 6321e9f767SBen Widawsky /* Check again in case preempted before timeout test */ 645e2411aeSIra Weiny if (!cxl_doorbell_busy(cxlds)) 6521e9f767SBen Widawsky break; 6621e9f767SBen Widawsky return -ETIMEDOUT; 6721e9f767SBen Widawsky } 6821e9f767SBen Widawsky cpu_relax(); 6921e9f767SBen Widawsky } 7021e9f767SBen Widawsky 715e2411aeSIra Weiny dev_dbg(cxlds->dev, "Doorbell wait took %dms", 7221e9f767SBen Widawsky jiffies_to_msecs(end) - jiffies_to_msecs(start)); 7321e9f767SBen Widawsky return 0; 7421e9f767SBen Widawsky } 7521e9f767SBen Widawsky 764f195ee7SDan Williams #define cxl_err(dev, status, msg) \ 774f195ee7SDan Williams dev_err_ratelimited(dev, msg ", device state %s%s\n", \ 784f195ee7SDan Williams status & CXLMDEV_DEV_FATAL ? " fatal" : "", \ 794f195ee7SDan Williams status & CXLMDEV_FW_HALT ? " firmware-halt" : "") 8021e9f767SBen Widawsky 814f195ee7SDan Williams #define cxl_cmd_err(dev, cmd, status, msg) \ 824f195ee7SDan Williams dev_err_ratelimited(dev, msg " (opcode: %#x), device state %s%s\n", \ 834f195ee7SDan Williams (cmd)->opcode, \ 844f195ee7SDan Williams status & CXLMDEV_DEV_FATAL ? " fatal" : "", \ 854f195ee7SDan Williams status & CXLMDEV_FW_HALT ? " firmware-halt" : "") 8621e9f767SBen Widawsky 8721e9f767SBen Widawsky /** 88ed97afb5SBen Widawsky * __cxl_pci_mbox_send_cmd() - Execute a mailbox command 895e2411aeSIra Weiny * @cxlds: The device state to communicate with. 9021e9f767SBen Widawsky * @mbox_cmd: Command to send to the memory device. 9121e9f767SBen Widawsky * 9221e9f767SBen Widawsky * Context: Any context. Expects mbox_mutex to be held. 9321e9f767SBen Widawsky * Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success. 9421e9f767SBen Widawsky * Caller should check the return code in @mbox_cmd to make sure it 9521e9f767SBen Widawsky * succeeded. 9621e9f767SBen Widawsky * 9721e9f767SBen Widawsky * This is a generic form of the CXL mailbox send command thus only using the 9821e9f767SBen Widawsky * registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory 9921e9f767SBen Widawsky * devices, and perhaps other types of CXL devices may have further information 10021e9f767SBen Widawsky * available upon error conditions. Driver facilities wishing to send mailbox 10121e9f767SBen Widawsky * commands should use the wrapper command. 10221e9f767SBen Widawsky * 10321e9f767SBen Widawsky * The CXL spec allows for up to two mailboxes. The intention is for the primary 10421e9f767SBen Widawsky * mailbox to be OS controlled and the secondary mailbox to be used by system 10521e9f767SBen Widawsky * firmware. This allows the OS and firmware to communicate with the device and 10621e9f767SBen Widawsky * not need to coordinate with each other. The driver only uses the primary 10721e9f767SBen Widawsky * mailbox. 10821e9f767SBen Widawsky */ 1095e2411aeSIra Weiny static int __cxl_pci_mbox_send_cmd(struct cxl_dev_state *cxlds, 110b64955a9SDan Williams struct cxl_mbox_cmd *mbox_cmd) 11121e9f767SBen Widawsky { 1125e2411aeSIra Weiny void __iomem *payload = cxlds->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET; 1135e2411aeSIra Weiny struct device *dev = cxlds->dev; 11421e9f767SBen Widawsky u64 cmd_reg, status_reg; 11521e9f767SBen Widawsky size_t out_len; 11621e9f767SBen Widawsky int rc; 11721e9f767SBen Widawsky 1185e2411aeSIra Weiny lockdep_assert_held(&cxlds->mbox_mutex); 11921e9f767SBen Widawsky 12021e9f767SBen Widawsky /* 12121e9f767SBen Widawsky * Here are the steps from 8.2.8.4 of the CXL 2.0 spec. 12221e9f767SBen Widawsky * 1. Caller reads MB Control Register to verify doorbell is clear 12321e9f767SBen Widawsky * 2. Caller writes Command Register 12421e9f767SBen Widawsky * 3. Caller writes Command Payload Registers if input payload is non-empty 12521e9f767SBen Widawsky * 4. Caller writes MB Control Register to set doorbell 12621e9f767SBen Widawsky * 5. Caller either polls for doorbell to be clear or waits for interrupt if configured 12721e9f767SBen Widawsky * 6. Caller reads MB Status Register to fetch Return code 12821e9f767SBen Widawsky * 7. If command successful, Caller reads Command Register to get Payload Length 12921e9f767SBen Widawsky * 8. If output payload is non-empty, host reads Command Payload Registers 13021e9f767SBen Widawsky * 13121e9f767SBen Widawsky * Hardware is free to do whatever it wants before the doorbell is rung, 13221e9f767SBen Widawsky * and isn't allowed to change anything after it clears the doorbell. As 13321e9f767SBen Widawsky * such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can 13421e9f767SBen Widawsky * also happen in any order (though some orders might not make sense). 13521e9f767SBen Widawsky */ 13621e9f767SBen Widawsky 13721e9f767SBen Widawsky /* #1 */ 1385e2411aeSIra Weiny if (cxl_doorbell_busy(cxlds)) { 1394f195ee7SDan Williams u64 md_status = 1404f195ee7SDan Williams readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); 1414f195ee7SDan Williams 1424f195ee7SDan Williams cxl_cmd_err(cxlds->dev, mbox_cmd, md_status, 1434f195ee7SDan Williams "mailbox queue busy"); 14421e9f767SBen Widawsky return -EBUSY; 14521e9f767SBen Widawsky } 14621e9f767SBen Widawsky 14721e9f767SBen Widawsky cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK, 14821e9f767SBen Widawsky mbox_cmd->opcode); 14921e9f767SBen Widawsky if (mbox_cmd->size_in) { 15021e9f767SBen Widawsky if (WARN_ON(!mbox_cmd->payload_in)) 15121e9f767SBen Widawsky return -EINVAL; 15221e9f767SBen Widawsky 15321e9f767SBen Widawsky cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, 15421e9f767SBen Widawsky mbox_cmd->size_in); 15521e9f767SBen Widawsky memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in); 15621e9f767SBen Widawsky } 15721e9f767SBen Widawsky 15821e9f767SBen Widawsky /* #2, #3 */ 1595e2411aeSIra Weiny writeq(cmd_reg, cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 16021e9f767SBen Widawsky 16121e9f767SBen Widawsky /* #4 */ 16299e222a5SDan Williams dev_dbg(dev, "Sending command\n"); 16321e9f767SBen Widawsky writel(CXLDEV_MBOX_CTRL_DOORBELL, 1645e2411aeSIra Weiny cxlds->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET); 16521e9f767SBen Widawsky 16621e9f767SBen Widawsky /* #5 */ 1675e2411aeSIra Weiny rc = cxl_pci_mbox_wait_for_doorbell(cxlds); 16821e9f767SBen Widawsky if (rc == -ETIMEDOUT) { 1694f195ee7SDan Williams u64 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); 1704f195ee7SDan Williams 1714f195ee7SDan Williams cxl_cmd_err(cxlds->dev, mbox_cmd, md_status, "mailbox timeout"); 17221e9f767SBen Widawsky return rc; 17321e9f767SBen Widawsky } 17421e9f767SBen Widawsky 17521e9f767SBen Widawsky /* #6 */ 1765e2411aeSIra Weiny status_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET); 17721e9f767SBen Widawsky mbox_cmd->return_code = 17821e9f767SBen Widawsky FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg); 17921e9f767SBen Widawsky 18092fcc1abSDavidlohr Bueso if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS) { 181c43e036dSDavidlohr Bueso dev_dbg(dev, "Mailbox operation had an error: %s\n", 182c43e036dSDavidlohr Bueso cxl_mbox_cmd_rc2str(mbox_cmd)); 183cbe83a20SDavidlohr Bueso return 0; /* completed but caller must check return_code */ 18421e9f767SBen Widawsky } 18521e9f767SBen Widawsky 18621e9f767SBen Widawsky /* #7 */ 1875e2411aeSIra Weiny cmd_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 18821e9f767SBen Widawsky out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg); 18921e9f767SBen Widawsky 19021e9f767SBen Widawsky /* #8 */ 19121e9f767SBen Widawsky if (out_len && mbox_cmd->payload_out) { 19221e9f767SBen Widawsky /* 19321e9f767SBen Widawsky * Sanitize the copy. If hardware misbehaves, out_len per the 19421e9f767SBen Widawsky * spec can actually be greater than the max allowed size (21 19521e9f767SBen Widawsky * bits available but spec defined 1M max). The caller also may 19621e9f767SBen Widawsky * have requested less data than the hardware supplied even 19721e9f767SBen Widawsky * within spec. 19821e9f767SBen Widawsky */ 1995e2411aeSIra Weiny size_t n = min3(mbox_cmd->size_out, cxlds->payload_size, out_len); 20021e9f767SBen Widawsky 20121e9f767SBen Widawsky memcpy_fromio(mbox_cmd->payload_out, payload, n); 20221e9f767SBen Widawsky mbox_cmd->size_out = n; 20321e9f767SBen Widawsky } else { 20421e9f767SBen Widawsky mbox_cmd->size_out = 0; 20521e9f767SBen Widawsky } 20621e9f767SBen Widawsky 20721e9f767SBen Widawsky return 0; 20821e9f767SBen Widawsky } 20921e9f767SBen Widawsky 2105e2411aeSIra Weiny static int cxl_pci_mbox_send(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd) 211b64955a9SDan Williams { 212b64955a9SDan Williams int rc; 213b64955a9SDan Williams 2144f195ee7SDan Williams mutex_lock_io(&cxlds->mbox_mutex); 2155e2411aeSIra Weiny rc = __cxl_pci_mbox_send_cmd(cxlds, cmd); 2164f195ee7SDan Williams mutex_unlock(&cxlds->mbox_mutex); 217b64955a9SDan Williams 218b64955a9SDan Williams return rc; 219b64955a9SDan Williams } 220b64955a9SDan Williams 2215e2411aeSIra Weiny static int cxl_pci_setup_mailbox(struct cxl_dev_state *cxlds) 22221e9f767SBen Widawsky { 2235e2411aeSIra Weiny const int cap = readl(cxlds->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET); 224229e8828SBen Widawsky unsigned long timeout; 225229e8828SBen Widawsky u64 md_status; 226229e8828SBen Widawsky 227229e8828SBen Widawsky timeout = jiffies + mbox_ready_timeout * HZ; 228229e8828SBen Widawsky do { 229229e8828SBen Widawsky md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); 230229e8828SBen Widawsky if (md_status & CXLMDEV_MBOX_IF_READY) 231229e8828SBen Widawsky break; 232229e8828SBen Widawsky if (msleep_interruptible(100)) 233229e8828SBen Widawsky break; 234229e8828SBen Widawsky } while (!time_after(jiffies, timeout)); 235229e8828SBen Widawsky 236229e8828SBen Widawsky if (!(md_status & CXLMDEV_MBOX_IF_READY)) { 2374f195ee7SDan Williams cxl_err(cxlds->dev, md_status, 2384f195ee7SDan Williams "timeout awaiting mailbox ready"); 2394f195ee7SDan Williams return -ETIMEDOUT; 2404f195ee7SDan Williams } 2414f195ee7SDan Williams 2424f195ee7SDan Williams /* 2434f195ee7SDan Williams * A command may be in flight from a previous driver instance, 2444f195ee7SDan Williams * think kexec, do one doorbell wait so that 2454f195ee7SDan Williams * __cxl_pci_mbox_send_cmd() can assume that it is the only 2464f195ee7SDan Williams * source for future doorbell busy events. 2474f195ee7SDan Williams */ 2484f195ee7SDan Williams if (cxl_pci_mbox_wait_for_doorbell(cxlds) != 0) { 2494f195ee7SDan Williams cxl_err(cxlds->dev, md_status, "timeout awaiting mailbox idle"); 2504f195ee7SDan Williams return -ETIMEDOUT; 251229e8828SBen Widawsky } 25221e9f767SBen Widawsky 2535e2411aeSIra Weiny cxlds->mbox_send = cxl_pci_mbox_send; 2545e2411aeSIra Weiny cxlds->payload_size = 25521e9f767SBen Widawsky 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap); 25621e9f767SBen Widawsky 25721e9f767SBen Widawsky /* 25821e9f767SBen Widawsky * CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register 25921e9f767SBen Widawsky * 26021e9f767SBen Widawsky * If the size is too small, mandatory commands will not work and so 26121e9f767SBen Widawsky * there's no point in going forward. If the size is too large, there's 26221e9f767SBen Widawsky * no harm is soft limiting it. 26321e9f767SBen Widawsky */ 2645e2411aeSIra Weiny cxlds->payload_size = min_t(size_t, cxlds->payload_size, SZ_1M); 2655e2411aeSIra Weiny if (cxlds->payload_size < 256) { 2665e2411aeSIra Weiny dev_err(cxlds->dev, "Mailbox is too small (%zub)", 2675e2411aeSIra Weiny cxlds->payload_size); 26821e9f767SBen Widawsky return -ENXIO; 26921e9f767SBen Widawsky } 27021e9f767SBen Widawsky 2715e2411aeSIra Weiny dev_dbg(cxlds->dev, "Mailbox payload sized %zu", 2725e2411aeSIra Weiny cxlds->payload_size); 27321e9f767SBen Widawsky 27421e9f767SBen Widawsky return 0; 27521e9f767SBen Widawsky } 27621e9f767SBen Widawsky 277a261e9a1SDan Williams static int cxl_map_regblock(struct pci_dev *pdev, struct cxl_register_map *map) 2781b0a1a2aSBen Widawsky { 2797dc7a64dSBen Widawsky struct device *dev = &pdev->dev; 2801b0a1a2aSBen Widawsky 281*6c7f4f1eSDan Williams map->base = ioremap(map->resource, map->max_size); 282*6c7f4f1eSDan Williams if (!map->base) { 28321e9f767SBen Widawsky dev_err(dev, "failed to map registers\n"); 284a261e9a1SDan Williams return -ENOMEM; 28521e9f767SBen Widawsky } 28621e9f767SBen Widawsky 287*6c7f4f1eSDan Williams dev_dbg(dev, "Mapped CXL Memory Device resource %pa\n", &map->resource); 288a261e9a1SDan Williams return 0; 28930af9729SIra Weiny } 29030af9729SIra Weiny 291a261e9a1SDan Williams static void cxl_unmap_regblock(struct pci_dev *pdev, 292a261e9a1SDan Williams struct cxl_register_map *map) 29330af9729SIra Weiny { 294*6c7f4f1eSDan Williams iounmap(map->base); 295a261e9a1SDan Williams map->base = NULL; 29621e9f767SBen Widawsky } 29721e9f767SBen Widawsky 298a261e9a1SDan Williams static int cxl_probe_regs(struct pci_dev *pdev, struct cxl_register_map *map) 29930af9729SIra Weiny { 30008422378SBen Widawsky struct cxl_component_reg_map *comp_map; 30130af9729SIra Weiny struct cxl_device_reg_map *dev_map; 3027dc7a64dSBen Widawsky struct device *dev = &pdev->dev; 303a261e9a1SDan Williams void __iomem *base = map->base; 30430af9729SIra Weiny 30530af9729SIra Weiny switch (map->reg_type) { 30608422378SBen Widawsky case CXL_REGLOC_RBI_COMPONENT: 30708422378SBen Widawsky comp_map = &map->component_map; 30808422378SBen Widawsky cxl_probe_component_regs(dev, base, comp_map); 30908422378SBen Widawsky if (!comp_map->hdm_decoder.valid) { 31008422378SBen Widawsky dev_err(dev, "HDM decoder registers not found\n"); 31108422378SBen Widawsky return -ENXIO; 31208422378SBen Widawsky } 31308422378SBen Widawsky 31408422378SBen Widawsky dev_dbg(dev, "Set up component registers\n"); 31508422378SBen Widawsky break; 31630af9729SIra Weiny case CXL_REGLOC_RBI_MEMDEV: 31730af9729SIra Weiny dev_map = &map->device_map; 31830af9729SIra Weiny cxl_probe_device_regs(dev, base, dev_map); 31930af9729SIra Weiny if (!dev_map->status.valid || !dev_map->mbox.valid || 32030af9729SIra Weiny !dev_map->memdev.valid) { 32130af9729SIra Weiny dev_err(dev, "registers not found: %s%s%s\n", 32230af9729SIra Weiny !dev_map->status.valid ? "status " : "", 323da582aa5SLi Qiang (Johnny Li) !dev_map->mbox.valid ? "mbox " : "", 324da582aa5SLi Qiang (Johnny Li) !dev_map->memdev.valid ? "memdev " : ""); 32530af9729SIra Weiny return -ENXIO; 32630af9729SIra Weiny } 32730af9729SIra Weiny 32830af9729SIra Weiny dev_dbg(dev, "Probing device registers...\n"); 32930af9729SIra Weiny break; 33030af9729SIra Weiny default: 33130af9729SIra Weiny break; 33230af9729SIra Weiny } 33330af9729SIra Weiny 33430af9729SIra Weiny return 0; 33530af9729SIra Weiny } 33630af9729SIra Weiny 33785afc317SBen Widawsky static int cxl_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type, 33885afc317SBen Widawsky struct cxl_register_map *map) 33985afc317SBen Widawsky { 34085afc317SBen Widawsky int rc; 34185afc317SBen Widawsky 34285afc317SBen Widawsky rc = cxl_find_regblock(pdev, type, map); 34385afc317SBen Widawsky if (rc) 34485afc317SBen Widawsky return rc; 34585afc317SBen Widawsky 34685afc317SBen Widawsky rc = cxl_map_regblock(pdev, map); 34785afc317SBen Widawsky if (rc) 34885afc317SBen Widawsky return rc; 34985afc317SBen Widawsky 35085afc317SBen Widawsky rc = cxl_probe_regs(pdev, map); 351a261e9a1SDan Williams cxl_unmap_regblock(pdev, map); 3525b68705dSBen Widawsky 35385afc317SBen Widawsky return rc; 3541d5a4159SBen Widawsky } 3551d5a4159SBen Widawsky 3563eddcc93SIra Weiny static void cxl_pci_destroy_doe(void *mbs) 3573eddcc93SIra Weiny { 3583eddcc93SIra Weiny xa_destroy(mbs); 3593eddcc93SIra Weiny } 3603eddcc93SIra Weiny 3613eddcc93SIra Weiny static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds) 3623eddcc93SIra Weiny { 3633eddcc93SIra Weiny struct device *dev = cxlds->dev; 3643eddcc93SIra Weiny struct pci_dev *pdev = to_pci_dev(dev); 3653eddcc93SIra Weiny u16 off = 0; 3663eddcc93SIra Weiny 3673eddcc93SIra Weiny xa_init(&cxlds->doe_mbs); 3683eddcc93SIra Weiny if (devm_add_action(&pdev->dev, cxl_pci_destroy_doe, &cxlds->doe_mbs)) { 3693eddcc93SIra Weiny dev_err(dev, "Failed to create XArray for DOE's\n"); 3703eddcc93SIra Weiny return; 3713eddcc93SIra Weiny } 3723eddcc93SIra Weiny 3733eddcc93SIra Weiny /* 3743eddcc93SIra Weiny * Mailbox creation is best effort. Higher layers must determine if 3753eddcc93SIra Weiny * the lack of a mailbox for their protocol is a device failure or not. 3763eddcc93SIra Weiny */ 3773eddcc93SIra Weiny pci_doe_for_each_off(pdev, off) { 3783eddcc93SIra Weiny struct pci_doe_mb *doe_mb; 3793eddcc93SIra Weiny 3803eddcc93SIra Weiny doe_mb = pcim_doe_create_mb(pdev, off); 3813eddcc93SIra Weiny if (IS_ERR(doe_mb)) { 3823eddcc93SIra Weiny dev_err(dev, "Failed to create MB object for MB @ %x\n", 3833eddcc93SIra Weiny off); 3843eddcc93SIra Weiny continue; 3853eddcc93SIra Weiny } 3863eddcc93SIra Weiny 3873eddcc93SIra Weiny if (xa_insert(&cxlds->doe_mbs, off, doe_mb, GFP_KERNEL)) { 3883eddcc93SIra Weiny dev_err(dev, "xa_insert failed to insert MB @ %x\n", 3893eddcc93SIra Weiny off); 3903eddcc93SIra Weiny continue; 3913eddcc93SIra Weiny } 3923eddcc93SIra Weiny 3933eddcc93SIra Weiny dev_dbg(dev, "Created DOE mailbox @%x\n", off); 3943eddcc93SIra Weiny } 3953eddcc93SIra Weiny } 3963eddcc93SIra Weiny 397ed97afb5SBen Widawsky static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 39821e9f767SBen Widawsky { 39985afc317SBen Widawsky struct cxl_register_map map; 40021083f51SDan Williams struct cxl_memdev *cxlmd; 4015e2411aeSIra Weiny struct cxl_dev_state *cxlds; 4021d5a4159SBen Widawsky int rc; 40321e9f767SBen Widawsky 4045a2328f4SDan Williams /* 4055a2328f4SDan Williams * Double check the anonymous union trickery in struct cxl_regs 4065a2328f4SDan Williams * FIXME switch to struct_group() 4075a2328f4SDan Williams */ 4085a2328f4SDan Williams BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) != 4095a2328f4SDan Williams offsetof(struct cxl_regs, device_regs.memdev)); 4105a2328f4SDan Williams 41121e9f767SBen Widawsky rc = pcim_enable_device(pdev); 41221e9f767SBen Widawsky if (rc) 41321e9f767SBen Widawsky return rc; 41421e9f767SBen Widawsky 4155e2411aeSIra Weiny cxlds = cxl_dev_state_create(&pdev->dev); 4165e2411aeSIra Weiny if (IS_ERR(cxlds)) 4175e2411aeSIra Weiny return PTR_ERR(cxlds); 4181b0a1a2aSBen Widawsky 419bcc79ea3SDan Williams cxlds->serial = pci_get_dsn(pdev); 42006e279e5SBen Widawsky cxlds->cxl_dvsec = pci_find_dvsec_capability( 42106e279e5SBen Widawsky pdev, PCI_DVSEC_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE); 42206e279e5SBen Widawsky if (!cxlds->cxl_dvsec) 42306e279e5SBen Widawsky dev_warn(&pdev->dev, 42406e279e5SBen Widawsky "Device DVSEC not present, skip CXL.mem init\n"); 42506e279e5SBen Widawsky 42685afc317SBen Widawsky rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map); 42785afc317SBen Widawsky if (rc) 42885afc317SBen Widawsky return rc; 42985afc317SBen Widawsky 430*6c7f4f1eSDan Williams rc = cxl_map_device_regs(&pdev->dev, &cxlds->regs.device_regs, &map); 43121e9f767SBen Widawsky if (rc) 43221e9f767SBen Widawsky return rc; 43321e9f767SBen Widawsky 4344112a08dSBen Widawsky /* 4354112a08dSBen Widawsky * If the component registers can't be found, the cxl_pci driver may 4364112a08dSBen Widawsky * still be useful for management functions so don't return an error. 4374112a08dSBen Widawsky */ 4384112a08dSBen Widawsky cxlds->component_reg_phys = CXL_RESOURCE_NONE; 4394112a08dSBen Widawsky rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, &map); 4404112a08dSBen Widawsky if (rc) 4414112a08dSBen Widawsky dev_warn(&pdev->dev, "No component registers (%d)\n", rc); 4424112a08dSBen Widawsky 443*6c7f4f1eSDan Williams cxlds->component_reg_phys = map.resource; 4444112a08dSBen Widawsky 4453eddcc93SIra Weiny devm_cxl_pci_create_doe(cxlds); 4463eddcc93SIra Weiny 4475e2411aeSIra Weiny rc = cxl_pci_setup_mailbox(cxlds); 44821e9f767SBen Widawsky if (rc) 44921e9f767SBen Widawsky return rc; 45021e9f767SBen Widawsky 4515e2411aeSIra Weiny rc = cxl_enumerate_cmds(cxlds); 45221e9f767SBen Widawsky if (rc) 45321e9f767SBen Widawsky return rc; 45421e9f767SBen Widawsky 4555e2411aeSIra Weiny rc = cxl_dev_state_identify(cxlds); 45621e9f767SBen Widawsky if (rc) 45721e9f767SBen Widawsky return rc; 45821e9f767SBen Widawsky 4595e2411aeSIra Weiny rc = cxl_mem_create_range_info(cxlds); 460f847502aSIra Weiny if (rc) 461f847502aSIra Weiny return rc; 462f847502aSIra Weiny 4635e2411aeSIra Weiny cxlmd = devm_cxl_add_memdev(cxlds); 46421083f51SDan Williams if (IS_ERR(cxlmd)) 46521083f51SDan Williams return PTR_ERR(cxlmd); 46621083f51SDan Williams 467d3b75029SDan Williams if (resource_size(&cxlds->pmem_res) && IS_ENABLED(CONFIG_CXL_PMEM)) 46821083f51SDan Williams rc = devm_cxl_add_nvdimm(&pdev->dev, cxlmd); 46921083f51SDan Williams 47021083f51SDan Williams return rc; 47121e9f767SBen Widawsky } 47221e9f767SBen Widawsky 47321e9f767SBen Widawsky static const struct pci_device_id cxl_mem_pci_tbl[] = { 47421e9f767SBen Widawsky /* PCI class code for CXL.mem Type-3 Devices */ 47521e9f767SBen Widawsky { PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)}, 47621e9f767SBen Widawsky { /* terminate list */ }, 47721e9f767SBen Widawsky }; 47821e9f767SBen Widawsky MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl); 47921e9f767SBen Widawsky 480ed97afb5SBen Widawsky static struct pci_driver cxl_pci_driver = { 48121e9f767SBen Widawsky .name = KBUILD_MODNAME, 48221e9f767SBen Widawsky .id_table = cxl_mem_pci_tbl, 483ed97afb5SBen Widawsky .probe = cxl_pci_probe, 48421e9f767SBen Widawsky .driver = { 48521e9f767SBen Widawsky .probe_type = PROBE_PREFER_ASYNCHRONOUS, 48621e9f767SBen Widawsky }, 48721e9f767SBen Widawsky }; 48821e9f767SBen Widawsky 48921e9f767SBen Widawsky MODULE_LICENSE("GPL v2"); 490ed97afb5SBen Widawsky module_pci_driver(cxl_pci_driver); 49121e9f767SBen Widawsky MODULE_IMPORT_NS(CXL); 492