121e9f767SBen Widawsky // SPDX-License-Identifier: GPL-2.0-only 221e9f767SBen Widawsky /* Copyright(c) 2020 Intel Corporation. All rights reserved. */ 34faf31b4SDan Williams #include <linux/io-64-nonatomic-lo-hi.h> 4229e8828SBen Widawsky #include <linux/moduleparam.h> 521e9f767SBen Widawsky #include <linux/module.h> 6229e8828SBen Widawsky #include <linux/delay.h> 721e9f767SBen Widawsky #include <linux/sizes.h> 821e9f767SBen Widawsky #include <linux/mutex.h> 930af9729SIra Weiny #include <linux/list.h> 1021e9f767SBen Widawsky #include <linux/pci.h> 113eddcc93SIra Weiny #include <linux/pci-doe.h> 1221e9f767SBen Widawsky #include <linux/io.h> 135161a55cSBen Widawsky #include "cxlmem.h" 14af9cae9fSDan Williams #include "cxlpci.h" 1521e9f767SBen Widawsky #include "cxl.h" 1621e9f767SBen Widawsky 1721e9f767SBen Widawsky /** 1821e9f767SBen Widawsky * DOC: cxl pci 1921e9f767SBen Widawsky * 2021e9f767SBen Widawsky * This implements the PCI exclusive functionality for a CXL device as it is 2121e9f767SBen Widawsky * defined by the Compute Express Link specification. CXL devices may surface 22ed97afb5SBen Widawsky * certain functionality even if it isn't CXL enabled. While this driver is 23ed97afb5SBen Widawsky * focused around the PCI specific aspects of a CXL device, it binds to the 24ed97afb5SBen Widawsky * specific CXL memory device class code, and therefore the implementation of 25ed97afb5SBen Widawsky * cxl_pci is focused around CXL memory devices. 2621e9f767SBen Widawsky * 2721e9f767SBen Widawsky * The driver has several responsibilities, mainly: 2821e9f767SBen Widawsky * - Create the memX device and register on the CXL bus. 2921e9f767SBen Widawsky * - Enumerate device's register interface and map them. 30ed97afb5SBen Widawsky * - Registers nvdimm bridge device with cxl_core. 31ed97afb5SBen Widawsky * - Registers a CXL mailbox with cxl_core. 3221e9f767SBen Widawsky */ 3321e9f767SBen Widawsky 345e2411aeSIra Weiny #define cxl_doorbell_busy(cxlds) \ 355e2411aeSIra Weiny (readl((cxlds)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \ 3621e9f767SBen Widawsky CXLDEV_MBOX_CTRL_DOORBELL) 3721e9f767SBen Widawsky 3821e9f767SBen Widawsky /* CXL 2.0 - 8.2.8.4 */ 3921e9f767SBen Widawsky #define CXL_MAILBOX_TIMEOUT_MS (2 * HZ) 4021e9f767SBen Widawsky 41229e8828SBen Widawsky /* 42229e8828SBen Widawsky * CXL 2.0 ECN "Add Mailbox Ready Time" defines a capability field to 43229e8828SBen Widawsky * dictate how long to wait for the mailbox to become ready. The new 44229e8828SBen Widawsky * field allows the device to tell software the amount of time to wait 45229e8828SBen Widawsky * before mailbox ready. This field per the spec theoretically allows 46229e8828SBen Widawsky * for up to 255 seconds. 255 seconds is unreasonably long, its longer 47229e8828SBen Widawsky * than the maximum SATA port link recovery wait. Default to 60 seconds 48229e8828SBen Widawsky * until someone builds a CXL device that needs more time in practice. 49229e8828SBen Widawsky */ 50229e8828SBen Widawsky static unsigned short mbox_ready_timeout = 60; 51229e8828SBen Widawsky module_param(mbox_ready_timeout, ushort, 0644); 522e4ba0ecSDan Williams MODULE_PARM_DESC(mbox_ready_timeout, "seconds to wait for mailbox ready"); 53229e8828SBen Widawsky 545e2411aeSIra Weiny static int cxl_pci_mbox_wait_for_doorbell(struct cxl_dev_state *cxlds) 5521e9f767SBen Widawsky { 5621e9f767SBen Widawsky const unsigned long start = jiffies; 5721e9f767SBen Widawsky unsigned long end = start; 5821e9f767SBen Widawsky 595e2411aeSIra Weiny while (cxl_doorbell_busy(cxlds)) { 6021e9f767SBen Widawsky end = jiffies; 6121e9f767SBen Widawsky 6221e9f767SBen Widawsky if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) { 6321e9f767SBen Widawsky /* Check again in case preempted before timeout test */ 645e2411aeSIra Weiny if (!cxl_doorbell_busy(cxlds)) 6521e9f767SBen Widawsky break; 6621e9f767SBen Widawsky return -ETIMEDOUT; 6721e9f767SBen Widawsky } 6821e9f767SBen Widawsky cpu_relax(); 6921e9f767SBen Widawsky } 7021e9f767SBen Widawsky 715e2411aeSIra Weiny dev_dbg(cxlds->dev, "Doorbell wait took %dms", 7221e9f767SBen Widawsky jiffies_to_msecs(end) - jiffies_to_msecs(start)); 7321e9f767SBen Widawsky return 0; 7421e9f767SBen Widawsky } 7521e9f767SBen Widawsky 764f195ee7SDan Williams #define cxl_err(dev, status, msg) \ 774f195ee7SDan Williams dev_err_ratelimited(dev, msg ", device state %s%s\n", \ 784f195ee7SDan Williams status & CXLMDEV_DEV_FATAL ? " fatal" : "", \ 794f195ee7SDan Williams status & CXLMDEV_FW_HALT ? " firmware-halt" : "") 8021e9f767SBen Widawsky 814f195ee7SDan Williams #define cxl_cmd_err(dev, cmd, status, msg) \ 824f195ee7SDan Williams dev_err_ratelimited(dev, msg " (opcode: %#x), device state %s%s\n", \ 834f195ee7SDan Williams (cmd)->opcode, \ 844f195ee7SDan Williams status & CXLMDEV_DEV_FATAL ? " fatal" : "", \ 854f195ee7SDan Williams status & CXLMDEV_FW_HALT ? " firmware-halt" : "") 8621e9f767SBen Widawsky 8721e9f767SBen Widawsky /** 88ed97afb5SBen Widawsky * __cxl_pci_mbox_send_cmd() - Execute a mailbox command 895e2411aeSIra Weiny * @cxlds: The device state to communicate with. 9021e9f767SBen Widawsky * @mbox_cmd: Command to send to the memory device. 9121e9f767SBen Widawsky * 9221e9f767SBen Widawsky * Context: Any context. Expects mbox_mutex to be held. 9321e9f767SBen Widawsky * Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success. 9421e9f767SBen Widawsky * Caller should check the return code in @mbox_cmd to make sure it 9521e9f767SBen Widawsky * succeeded. 9621e9f767SBen Widawsky * 9721e9f767SBen Widawsky * This is a generic form of the CXL mailbox send command thus only using the 9821e9f767SBen Widawsky * registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory 9921e9f767SBen Widawsky * devices, and perhaps other types of CXL devices may have further information 10021e9f767SBen Widawsky * available upon error conditions. Driver facilities wishing to send mailbox 10121e9f767SBen Widawsky * commands should use the wrapper command. 10221e9f767SBen Widawsky * 10321e9f767SBen Widawsky * The CXL spec allows for up to two mailboxes. The intention is for the primary 10421e9f767SBen Widawsky * mailbox to be OS controlled and the secondary mailbox to be used by system 10521e9f767SBen Widawsky * firmware. This allows the OS and firmware to communicate with the device and 10621e9f767SBen Widawsky * not need to coordinate with each other. The driver only uses the primary 10721e9f767SBen Widawsky * mailbox. 10821e9f767SBen Widawsky */ 1095e2411aeSIra Weiny static int __cxl_pci_mbox_send_cmd(struct cxl_dev_state *cxlds, 110b64955a9SDan Williams struct cxl_mbox_cmd *mbox_cmd) 11121e9f767SBen Widawsky { 1125e2411aeSIra Weiny void __iomem *payload = cxlds->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET; 1135e2411aeSIra Weiny struct device *dev = cxlds->dev; 11421e9f767SBen Widawsky u64 cmd_reg, status_reg; 11521e9f767SBen Widawsky size_t out_len; 11621e9f767SBen Widawsky int rc; 11721e9f767SBen Widawsky 1185e2411aeSIra Weiny lockdep_assert_held(&cxlds->mbox_mutex); 11921e9f767SBen Widawsky 12021e9f767SBen Widawsky /* 12121e9f767SBen Widawsky * Here are the steps from 8.2.8.4 of the CXL 2.0 spec. 12221e9f767SBen Widawsky * 1. Caller reads MB Control Register to verify doorbell is clear 12321e9f767SBen Widawsky * 2. Caller writes Command Register 12421e9f767SBen Widawsky * 3. Caller writes Command Payload Registers if input payload is non-empty 12521e9f767SBen Widawsky * 4. Caller writes MB Control Register to set doorbell 12621e9f767SBen Widawsky * 5. Caller either polls for doorbell to be clear or waits for interrupt if configured 12721e9f767SBen Widawsky * 6. Caller reads MB Status Register to fetch Return code 12821e9f767SBen Widawsky * 7. If command successful, Caller reads Command Register to get Payload Length 12921e9f767SBen Widawsky * 8. If output payload is non-empty, host reads Command Payload Registers 13021e9f767SBen Widawsky * 13121e9f767SBen Widawsky * Hardware is free to do whatever it wants before the doorbell is rung, 13221e9f767SBen Widawsky * and isn't allowed to change anything after it clears the doorbell. As 13321e9f767SBen Widawsky * such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can 13421e9f767SBen Widawsky * also happen in any order (though some orders might not make sense). 13521e9f767SBen Widawsky */ 13621e9f767SBen Widawsky 13721e9f767SBen Widawsky /* #1 */ 1385e2411aeSIra Weiny if (cxl_doorbell_busy(cxlds)) { 1394f195ee7SDan Williams u64 md_status = 1404f195ee7SDan Williams readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); 1414f195ee7SDan Williams 1424f195ee7SDan Williams cxl_cmd_err(cxlds->dev, mbox_cmd, md_status, 1434f195ee7SDan Williams "mailbox queue busy"); 14421e9f767SBen Widawsky return -EBUSY; 14521e9f767SBen Widawsky } 14621e9f767SBen Widawsky 14721e9f767SBen Widawsky cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK, 14821e9f767SBen Widawsky mbox_cmd->opcode); 14921e9f767SBen Widawsky if (mbox_cmd->size_in) { 15021e9f767SBen Widawsky if (WARN_ON(!mbox_cmd->payload_in)) 15121e9f767SBen Widawsky return -EINVAL; 15221e9f767SBen Widawsky 15321e9f767SBen Widawsky cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, 15421e9f767SBen Widawsky mbox_cmd->size_in); 15521e9f767SBen Widawsky memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in); 15621e9f767SBen Widawsky } 15721e9f767SBen Widawsky 15821e9f767SBen Widawsky /* #2, #3 */ 1595e2411aeSIra Weiny writeq(cmd_reg, cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 16021e9f767SBen Widawsky 16121e9f767SBen Widawsky /* #4 */ 16299e222a5SDan Williams dev_dbg(dev, "Sending command\n"); 16321e9f767SBen Widawsky writel(CXLDEV_MBOX_CTRL_DOORBELL, 1645e2411aeSIra Weiny cxlds->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET); 16521e9f767SBen Widawsky 16621e9f767SBen Widawsky /* #5 */ 1675e2411aeSIra Weiny rc = cxl_pci_mbox_wait_for_doorbell(cxlds); 16821e9f767SBen Widawsky if (rc == -ETIMEDOUT) { 1694f195ee7SDan Williams u64 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); 1704f195ee7SDan Williams 1714f195ee7SDan Williams cxl_cmd_err(cxlds->dev, mbox_cmd, md_status, "mailbox timeout"); 17221e9f767SBen Widawsky return rc; 17321e9f767SBen Widawsky } 17421e9f767SBen Widawsky 17521e9f767SBen Widawsky /* #6 */ 1765e2411aeSIra Weiny status_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET); 17721e9f767SBen Widawsky mbox_cmd->return_code = 17821e9f767SBen Widawsky FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg); 17921e9f767SBen Widawsky 18092fcc1abSDavidlohr Bueso if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS) { 181c43e036dSDavidlohr Bueso dev_dbg(dev, "Mailbox operation had an error: %s\n", 182c43e036dSDavidlohr Bueso cxl_mbox_cmd_rc2str(mbox_cmd)); 183cbe83a20SDavidlohr Bueso return 0; /* completed but caller must check return_code */ 18421e9f767SBen Widawsky } 18521e9f767SBen Widawsky 18621e9f767SBen Widawsky /* #7 */ 1875e2411aeSIra Weiny cmd_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 18821e9f767SBen Widawsky out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg); 18921e9f767SBen Widawsky 19021e9f767SBen Widawsky /* #8 */ 19121e9f767SBen Widawsky if (out_len && mbox_cmd->payload_out) { 19221e9f767SBen Widawsky /* 19321e9f767SBen Widawsky * Sanitize the copy. If hardware misbehaves, out_len per the 19421e9f767SBen Widawsky * spec can actually be greater than the max allowed size (21 19521e9f767SBen Widawsky * bits available but spec defined 1M max). The caller also may 19621e9f767SBen Widawsky * have requested less data than the hardware supplied even 19721e9f767SBen Widawsky * within spec. 19821e9f767SBen Widawsky */ 1995e2411aeSIra Weiny size_t n = min3(mbox_cmd->size_out, cxlds->payload_size, out_len); 20021e9f767SBen Widawsky 20121e9f767SBen Widawsky memcpy_fromio(mbox_cmd->payload_out, payload, n); 20221e9f767SBen Widawsky mbox_cmd->size_out = n; 20321e9f767SBen Widawsky } else { 20421e9f767SBen Widawsky mbox_cmd->size_out = 0; 20521e9f767SBen Widawsky } 20621e9f767SBen Widawsky 20721e9f767SBen Widawsky return 0; 20821e9f767SBen Widawsky } 20921e9f767SBen Widawsky 2105e2411aeSIra Weiny static int cxl_pci_mbox_send(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd) 211b64955a9SDan Williams { 212b64955a9SDan Williams int rc; 213b64955a9SDan Williams 2144f195ee7SDan Williams mutex_lock_io(&cxlds->mbox_mutex); 2155e2411aeSIra Weiny rc = __cxl_pci_mbox_send_cmd(cxlds, cmd); 2164f195ee7SDan Williams mutex_unlock(&cxlds->mbox_mutex); 217b64955a9SDan Williams 218b64955a9SDan Williams return rc; 219b64955a9SDan Williams } 220b64955a9SDan Williams 2215e2411aeSIra Weiny static int cxl_pci_setup_mailbox(struct cxl_dev_state *cxlds) 22221e9f767SBen Widawsky { 2235e2411aeSIra Weiny const int cap = readl(cxlds->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET); 224229e8828SBen Widawsky unsigned long timeout; 225229e8828SBen Widawsky u64 md_status; 226229e8828SBen Widawsky 227229e8828SBen Widawsky timeout = jiffies + mbox_ready_timeout * HZ; 228229e8828SBen Widawsky do { 229229e8828SBen Widawsky md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); 230229e8828SBen Widawsky if (md_status & CXLMDEV_MBOX_IF_READY) 231229e8828SBen Widawsky break; 232229e8828SBen Widawsky if (msleep_interruptible(100)) 233229e8828SBen Widawsky break; 234229e8828SBen Widawsky } while (!time_after(jiffies, timeout)); 235229e8828SBen Widawsky 236229e8828SBen Widawsky if (!(md_status & CXLMDEV_MBOX_IF_READY)) { 2374f195ee7SDan Williams cxl_err(cxlds->dev, md_status, 2384f195ee7SDan Williams "timeout awaiting mailbox ready"); 2394f195ee7SDan Williams return -ETIMEDOUT; 2404f195ee7SDan Williams } 2414f195ee7SDan Williams 2424f195ee7SDan Williams /* 2434f195ee7SDan Williams * A command may be in flight from a previous driver instance, 2444f195ee7SDan Williams * think kexec, do one doorbell wait so that 2454f195ee7SDan Williams * __cxl_pci_mbox_send_cmd() can assume that it is the only 2464f195ee7SDan Williams * source for future doorbell busy events. 2474f195ee7SDan Williams */ 2484f195ee7SDan Williams if (cxl_pci_mbox_wait_for_doorbell(cxlds) != 0) { 2494f195ee7SDan Williams cxl_err(cxlds->dev, md_status, "timeout awaiting mailbox idle"); 2504f195ee7SDan Williams return -ETIMEDOUT; 251229e8828SBen Widawsky } 25221e9f767SBen Widawsky 2535e2411aeSIra Weiny cxlds->mbox_send = cxl_pci_mbox_send; 2545e2411aeSIra Weiny cxlds->payload_size = 25521e9f767SBen Widawsky 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap); 25621e9f767SBen Widawsky 25721e9f767SBen Widawsky /* 25821e9f767SBen Widawsky * CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register 25921e9f767SBen Widawsky * 26021e9f767SBen Widawsky * If the size is too small, mandatory commands will not work and so 26121e9f767SBen Widawsky * there's no point in going forward. If the size is too large, there's 26221e9f767SBen Widawsky * no harm is soft limiting it. 26321e9f767SBen Widawsky */ 2645e2411aeSIra Weiny cxlds->payload_size = min_t(size_t, cxlds->payload_size, SZ_1M); 2655e2411aeSIra Weiny if (cxlds->payload_size < 256) { 2665e2411aeSIra Weiny dev_err(cxlds->dev, "Mailbox is too small (%zub)", 2675e2411aeSIra Weiny cxlds->payload_size); 26821e9f767SBen Widawsky return -ENXIO; 26921e9f767SBen Widawsky } 27021e9f767SBen Widawsky 2715e2411aeSIra Weiny dev_dbg(cxlds->dev, "Mailbox payload sized %zu", 2725e2411aeSIra Weiny cxlds->payload_size); 27321e9f767SBen Widawsky 27421e9f767SBen Widawsky return 0; 27521e9f767SBen Widawsky } 27621e9f767SBen Widawsky 277a261e9a1SDan Williams static int cxl_map_regblock(struct pci_dev *pdev, struct cxl_register_map *map) 2781b0a1a2aSBen Widawsky { 279f8a7e8c2SIra Weiny void __iomem *addr; 2807dc7a64dSBen Widawsky int bar = map->barno; 2817dc7a64dSBen Widawsky struct device *dev = &pdev->dev; 2827dc7a64dSBen Widawsky resource_size_t offset = map->block_offset; 2831b0a1a2aSBen Widawsky 28421e9f767SBen Widawsky /* Basic sanity check that BAR is big enough */ 28521e9f767SBen Widawsky if (pci_resource_len(pdev, bar) < offset) { 2867dc7a64dSBen Widawsky dev_err(dev, "BAR%d: %pr: too small (offset: %pa)\n", bar, 2877dc7a64dSBen Widawsky &pdev->resource[bar], &offset); 288a261e9a1SDan Williams return -ENXIO; 28921e9f767SBen Widawsky } 29021e9f767SBen Widawsky 29130af9729SIra Weiny addr = pci_iomap(pdev, bar, 0); 292f8a7e8c2SIra Weiny if (!addr) { 29321e9f767SBen Widawsky dev_err(dev, "failed to map registers\n"); 294a261e9a1SDan Williams return -ENOMEM; 29521e9f767SBen Widawsky } 29621e9f767SBen Widawsky 2977dc7a64dSBen Widawsky dev_dbg(dev, "Mapped CXL Memory Device resource bar %u @ %pa\n", 2987dc7a64dSBen Widawsky bar, &offset); 2996630d31cSBen Widawsky 300a261e9a1SDan Williams map->base = addr + map->block_offset; 301a261e9a1SDan Williams return 0; 30230af9729SIra Weiny } 30330af9729SIra Weiny 304a261e9a1SDan Williams static void cxl_unmap_regblock(struct pci_dev *pdev, 305a261e9a1SDan Williams struct cxl_register_map *map) 30630af9729SIra Weiny { 307a261e9a1SDan Williams pci_iounmap(pdev, map->base - map->block_offset); 308a261e9a1SDan Williams map->base = NULL; 30921e9f767SBen Widawsky } 31021e9f767SBen Widawsky 311a261e9a1SDan Williams static int cxl_probe_regs(struct pci_dev *pdev, struct cxl_register_map *map) 31230af9729SIra Weiny { 31308422378SBen Widawsky struct cxl_component_reg_map *comp_map; 31430af9729SIra Weiny struct cxl_device_reg_map *dev_map; 3157dc7a64dSBen Widawsky struct device *dev = &pdev->dev; 316a261e9a1SDan Williams void __iomem *base = map->base; 31730af9729SIra Weiny 31830af9729SIra Weiny switch (map->reg_type) { 31908422378SBen Widawsky case CXL_REGLOC_RBI_COMPONENT: 32008422378SBen Widawsky comp_map = &map->component_map; 32108422378SBen Widawsky cxl_probe_component_regs(dev, base, comp_map); 32208422378SBen Widawsky if (!comp_map->hdm_decoder.valid) { 32308422378SBen Widawsky dev_err(dev, "HDM decoder registers not found\n"); 32408422378SBen Widawsky return -ENXIO; 32508422378SBen Widawsky } 32608422378SBen Widawsky 32708422378SBen Widawsky dev_dbg(dev, "Set up component registers\n"); 32808422378SBen Widawsky break; 32930af9729SIra Weiny case CXL_REGLOC_RBI_MEMDEV: 33030af9729SIra Weiny dev_map = &map->device_map; 33130af9729SIra Weiny cxl_probe_device_regs(dev, base, dev_map); 33230af9729SIra Weiny if (!dev_map->status.valid || !dev_map->mbox.valid || 33330af9729SIra Weiny !dev_map->memdev.valid) { 33430af9729SIra Weiny dev_err(dev, "registers not found: %s%s%s\n", 33530af9729SIra Weiny !dev_map->status.valid ? "status " : "", 336da582aa5SLi Qiang (Johnny Li) !dev_map->mbox.valid ? "mbox " : "", 337da582aa5SLi Qiang (Johnny Li) !dev_map->memdev.valid ? "memdev " : ""); 33830af9729SIra Weiny return -ENXIO; 33930af9729SIra Weiny } 34030af9729SIra Weiny 34130af9729SIra Weiny dev_dbg(dev, "Probing device registers...\n"); 34230af9729SIra Weiny break; 34330af9729SIra Weiny default: 34430af9729SIra Weiny break; 34530af9729SIra Weiny } 34630af9729SIra Weiny 34730af9729SIra Weiny return 0; 34830af9729SIra Weiny } 34930af9729SIra Weiny 3505e2411aeSIra Weiny static int cxl_map_regs(struct cxl_dev_state *cxlds, struct cxl_register_map *map) 35130af9729SIra Weiny { 3525e2411aeSIra Weiny struct device *dev = cxlds->dev; 35399e222a5SDan Williams struct pci_dev *pdev = to_pci_dev(dev); 35430af9729SIra Weiny 35530af9729SIra Weiny switch (map->reg_type) { 35608422378SBen Widawsky case CXL_REGLOC_RBI_COMPONENT: 3575e2411aeSIra Weiny cxl_map_component_regs(pdev, &cxlds->regs.component, map); 35808422378SBen Widawsky dev_dbg(dev, "Mapping component registers...\n"); 35908422378SBen Widawsky break; 36030af9729SIra Weiny case CXL_REGLOC_RBI_MEMDEV: 3615e2411aeSIra Weiny cxl_map_device_regs(pdev, &cxlds->regs.device_regs, map); 36230af9729SIra Weiny dev_dbg(dev, "Probing device registers...\n"); 36330af9729SIra Weiny break; 36430af9729SIra Weiny default: 36530af9729SIra Weiny break; 36630af9729SIra Weiny } 36730af9729SIra Weiny 36830af9729SIra Weiny return 0; 36930af9729SIra Weiny } 37030af9729SIra Weiny 37185afc317SBen Widawsky static int cxl_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type, 37285afc317SBen Widawsky struct cxl_register_map *map) 37385afc317SBen Widawsky { 37485afc317SBen Widawsky int rc; 37585afc317SBen Widawsky 37685afc317SBen Widawsky rc = cxl_find_regblock(pdev, type, map); 37785afc317SBen Widawsky if (rc) 37885afc317SBen Widawsky return rc; 37985afc317SBen Widawsky 38085afc317SBen Widawsky rc = cxl_map_regblock(pdev, map); 38185afc317SBen Widawsky if (rc) 38285afc317SBen Widawsky return rc; 38385afc317SBen Widawsky 38485afc317SBen Widawsky rc = cxl_probe_regs(pdev, map); 385a261e9a1SDan Williams cxl_unmap_regblock(pdev, map); 3865b68705dSBen Widawsky 38785afc317SBen Widawsky return rc; 3881d5a4159SBen Widawsky } 3891d5a4159SBen Widawsky 3903eddcc93SIra Weiny static void cxl_pci_destroy_doe(void *mbs) 3913eddcc93SIra Weiny { 3923eddcc93SIra Weiny xa_destroy(mbs); 3933eddcc93SIra Weiny } 3943eddcc93SIra Weiny 3953eddcc93SIra Weiny static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds) 3963eddcc93SIra Weiny { 3973eddcc93SIra Weiny struct device *dev = cxlds->dev; 3983eddcc93SIra Weiny struct pci_dev *pdev = to_pci_dev(dev); 3993eddcc93SIra Weiny u16 off = 0; 4003eddcc93SIra Weiny 4013eddcc93SIra Weiny xa_init(&cxlds->doe_mbs); 4023eddcc93SIra Weiny if (devm_add_action(&pdev->dev, cxl_pci_destroy_doe, &cxlds->doe_mbs)) { 4033eddcc93SIra Weiny dev_err(dev, "Failed to create XArray for DOE's\n"); 4043eddcc93SIra Weiny return; 4053eddcc93SIra Weiny } 4063eddcc93SIra Weiny 4073eddcc93SIra Weiny /* 4083eddcc93SIra Weiny * Mailbox creation is best effort. Higher layers must determine if 4093eddcc93SIra Weiny * the lack of a mailbox for their protocol is a device failure or not. 4103eddcc93SIra Weiny */ 4113eddcc93SIra Weiny pci_doe_for_each_off(pdev, off) { 4123eddcc93SIra Weiny struct pci_doe_mb *doe_mb; 4133eddcc93SIra Weiny 4143eddcc93SIra Weiny doe_mb = pcim_doe_create_mb(pdev, off); 4153eddcc93SIra Weiny if (IS_ERR(doe_mb)) { 4163eddcc93SIra Weiny dev_err(dev, "Failed to create MB object for MB @ %x\n", 4173eddcc93SIra Weiny off); 4183eddcc93SIra Weiny continue; 4193eddcc93SIra Weiny } 4203eddcc93SIra Weiny 421487d828dSIra Weiny if (!pci_request_config_region_exclusive(pdev, off, 422487d828dSIra Weiny PCI_DOE_CAP_SIZEOF, 423487d828dSIra Weiny dev_name(dev))) 424487d828dSIra Weiny pci_err(pdev, "Failed to exclude DOE registers\n"); 425487d828dSIra Weiny 4263eddcc93SIra Weiny if (xa_insert(&cxlds->doe_mbs, off, doe_mb, GFP_KERNEL)) { 4273eddcc93SIra Weiny dev_err(dev, "xa_insert failed to insert MB @ %x\n", 4283eddcc93SIra Weiny off); 4293eddcc93SIra Weiny continue; 4303eddcc93SIra Weiny } 4313eddcc93SIra Weiny 4323eddcc93SIra Weiny dev_dbg(dev, "Created DOE mailbox @%x\n", off); 4333eddcc93SIra Weiny } 4343eddcc93SIra Weiny } 4353eddcc93SIra Weiny 436*0a19bfc8SDan Williams /* 437*0a19bfc8SDan Williams * Assume that any RCIEP that emits the CXL memory expander class code 438*0a19bfc8SDan Williams * is an RCD 439*0a19bfc8SDan Williams */ 440*0a19bfc8SDan Williams static bool is_cxl_restricted(struct pci_dev *pdev) 441*0a19bfc8SDan Williams { 442*0a19bfc8SDan Williams return pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_END; 443*0a19bfc8SDan Williams } 444*0a19bfc8SDan Williams 445ed97afb5SBen Widawsky static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 44621e9f767SBen Widawsky { 44785afc317SBen Widawsky struct cxl_register_map map; 44821083f51SDan Williams struct cxl_memdev *cxlmd; 4495e2411aeSIra Weiny struct cxl_dev_state *cxlds; 4501d5a4159SBen Widawsky int rc; 45121e9f767SBen Widawsky 4525a2328f4SDan Williams /* 4535a2328f4SDan Williams * Double check the anonymous union trickery in struct cxl_regs 4545a2328f4SDan Williams * FIXME switch to struct_group() 4555a2328f4SDan Williams */ 4565a2328f4SDan Williams BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) != 4575a2328f4SDan Williams offsetof(struct cxl_regs, device_regs.memdev)); 4585a2328f4SDan Williams 45921e9f767SBen Widawsky rc = pcim_enable_device(pdev); 46021e9f767SBen Widawsky if (rc) 46121e9f767SBen Widawsky return rc; 46221e9f767SBen Widawsky 4635e2411aeSIra Weiny cxlds = cxl_dev_state_create(&pdev->dev); 4645e2411aeSIra Weiny if (IS_ERR(cxlds)) 4655e2411aeSIra Weiny return PTR_ERR(cxlds); 4661b0a1a2aSBen Widawsky 467*0a19bfc8SDan Williams cxlds->rcd = is_cxl_restricted(pdev); 468bcc79ea3SDan Williams cxlds->serial = pci_get_dsn(pdev); 46906e279e5SBen Widawsky cxlds->cxl_dvsec = pci_find_dvsec_capability( 47006e279e5SBen Widawsky pdev, PCI_DVSEC_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE); 47106e279e5SBen Widawsky if (!cxlds->cxl_dvsec) 47206e279e5SBen Widawsky dev_warn(&pdev->dev, 47306e279e5SBen Widawsky "Device DVSEC not present, skip CXL.mem init\n"); 47406e279e5SBen Widawsky 47585afc317SBen Widawsky rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map); 47685afc317SBen Widawsky if (rc) 47785afc317SBen Widawsky return rc; 47885afc317SBen Widawsky 4795e2411aeSIra Weiny rc = cxl_map_regs(cxlds, &map); 48021e9f767SBen Widawsky if (rc) 48121e9f767SBen Widawsky return rc; 48221e9f767SBen Widawsky 4834112a08dSBen Widawsky /* 4844112a08dSBen Widawsky * If the component registers can't be found, the cxl_pci driver may 4854112a08dSBen Widawsky * still be useful for management functions so don't return an error. 4864112a08dSBen Widawsky */ 4874112a08dSBen Widawsky cxlds->component_reg_phys = CXL_RESOURCE_NONE; 4884112a08dSBen Widawsky rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, &map); 4894112a08dSBen Widawsky if (rc) 4904112a08dSBen Widawsky dev_warn(&pdev->dev, "No component registers (%d)\n", rc); 4914112a08dSBen Widawsky 4924112a08dSBen Widawsky cxlds->component_reg_phys = cxl_regmap_to_base(pdev, &map); 4934112a08dSBen Widawsky 4943eddcc93SIra Weiny devm_cxl_pci_create_doe(cxlds); 4953eddcc93SIra Weiny 4965e2411aeSIra Weiny rc = cxl_pci_setup_mailbox(cxlds); 49721e9f767SBen Widawsky if (rc) 49821e9f767SBen Widawsky return rc; 49921e9f767SBen Widawsky 5005e2411aeSIra Weiny rc = cxl_enumerate_cmds(cxlds); 50121e9f767SBen Widawsky if (rc) 50221e9f767SBen Widawsky return rc; 50321e9f767SBen Widawsky 5045e2411aeSIra Weiny rc = cxl_dev_state_identify(cxlds); 50521e9f767SBen Widawsky if (rc) 50621e9f767SBen Widawsky return rc; 50721e9f767SBen Widawsky 5085e2411aeSIra Weiny rc = cxl_mem_create_range_info(cxlds); 509f847502aSIra Weiny if (rc) 510f847502aSIra Weiny return rc; 511f847502aSIra Weiny 5125e2411aeSIra Weiny cxlmd = devm_cxl_add_memdev(cxlds); 51321083f51SDan Williams if (IS_ERR(cxlmd)) 51421083f51SDan Williams return PTR_ERR(cxlmd); 51521083f51SDan Williams 51621083f51SDan Williams return rc; 51721e9f767SBen Widawsky } 51821e9f767SBen Widawsky 51921e9f767SBen Widawsky static const struct pci_device_id cxl_mem_pci_tbl[] = { 52021e9f767SBen Widawsky /* PCI class code for CXL.mem Type-3 Devices */ 52121e9f767SBen Widawsky { PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)}, 52221e9f767SBen Widawsky { /* terminate list */ }, 52321e9f767SBen Widawsky }; 52421e9f767SBen Widawsky MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl); 52521e9f767SBen Widawsky 526ed97afb5SBen Widawsky static struct pci_driver cxl_pci_driver = { 52721e9f767SBen Widawsky .name = KBUILD_MODNAME, 52821e9f767SBen Widawsky .id_table = cxl_mem_pci_tbl, 529ed97afb5SBen Widawsky .probe = cxl_pci_probe, 53021e9f767SBen Widawsky .driver = { 53121e9f767SBen Widawsky .probe_type = PROBE_PREFER_ASYNCHRONOUS, 53221e9f767SBen Widawsky }, 53321e9f767SBen Widawsky }; 53421e9f767SBen Widawsky 53521e9f767SBen Widawsky MODULE_LICENSE("GPL v2"); 536ed97afb5SBen Widawsky module_pci_driver(cxl_pci_driver); 53721e9f767SBen Widawsky MODULE_IMPORT_NS(CXL); 538