121e9f767SBen Widawsky // SPDX-License-Identifier: GPL-2.0-only 221e9f767SBen Widawsky /* Copyright(c) 2020 Intel Corporation. All rights reserved. */ 34faf31b4SDan Williams #include <linux/io-64-nonatomic-lo-hi.h> 421e9f767SBen Widawsky #include <linux/module.h> 521e9f767SBen Widawsky #include <linux/sizes.h> 621e9f767SBen Widawsky #include <linux/mutex.h> 730af9729SIra Weiny #include <linux/list.h> 821e9f767SBen Widawsky #include <linux/pci.h> 921e9f767SBen Widawsky #include <linux/io.h> 105161a55cSBen Widawsky #include "cxlmem.h" 1121e9f767SBen Widawsky #include "pci.h" 1221e9f767SBen Widawsky #include "cxl.h" 1321e9f767SBen Widawsky 1421e9f767SBen Widawsky /** 1521e9f767SBen Widawsky * DOC: cxl pci 1621e9f767SBen Widawsky * 1721e9f767SBen Widawsky * This implements the PCI exclusive functionality for a CXL device as it is 1821e9f767SBen Widawsky * defined by the Compute Express Link specification. CXL devices may surface 1921e9f767SBen Widawsky * certain functionality even if it isn't CXL enabled. 2021e9f767SBen Widawsky * 2121e9f767SBen Widawsky * The driver has several responsibilities, mainly: 2221e9f767SBen Widawsky * - Create the memX device and register on the CXL bus. 2321e9f767SBen Widawsky * - Enumerate device's register interface and map them. 2421e9f767SBen Widawsky * - Probe the device attributes to establish sysfs interface. 2521e9f767SBen Widawsky * - Provide an IOCTL interface to userspace to communicate with the device for 2621e9f767SBen Widawsky * things like firmware update. 2721e9f767SBen Widawsky */ 2821e9f767SBen Widawsky 2921e9f767SBen Widawsky #define cxl_doorbell_busy(cxlm) \ 3021e9f767SBen Widawsky (readl((cxlm)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \ 3121e9f767SBen Widawsky CXLDEV_MBOX_CTRL_DOORBELL) 3221e9f767SBen Widawsky 3321e9f767SBen Widawsky /* CXL 2.0 - 8.2.8.4 */ 3421e9f767SBen Widawsky #define CXL_MAILBOX_TIMEOUT_MS (2 * HZ) 3521e9f767SBen Widawsky 3621e9f767SBen Widawsky static int cxl_mem_wait_for_doorbell(struct cxl_mem *cxlm) 3721e9f767SBen Widawsky { 3821e9f767SBen Widawsky const unsigned long start = jiffies; 3921e9f767SBen Widawsky unsigned long end = start; 4021e9f767SBen Widawsky 4121e9f767SBen Widawsky while (cxl_doorbell_busy(cxlm)) { 4221e9f767SBen Widawsky end = jiffies; 4321e9f767SBen Widawsky 4421e9f767SBen Widawsky if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) { 4521e9f767SBen Widawsky /* Check again in case preempted before timeout test */ 4621e9f767SBen Widawsky if (!cxl_doorbell_busy(cxlm)) 4721e9f767SBen Widawsky break; 4821e9f767SBen Widawsky return -ETIMEDOUT; 4921e9f767SBen Widawsky } 5021e9f767SBen Widawsky cpu_relax(); 5121e9f767SBen Widawsky } 5221e9f767SBen Widawsky 5399e222a5SDan Williams dev_dbg(cxlm->dev, "Doorbell wait took %dms", 5421e9f767SBen Widawsky jiffies_to_msecs(end) - jiffies_to_msecs(start)); 5521e9f767SBen Widawsky return 0; 5621e9f767SBen Widawsky } 5721e9f767SBen Widawsky 5821e9f767SBen Widawsky static void cxl_mem_mbox_timeout(struct cxl_mem *cxlm, 59b64955a9SDan Williams struct cxl_mbox_cmd *mbox_cmd) 6021e9f767SBen Widawsky { 6199e222a5SDan Williams struct device *dev = cxlm->dev; 6221e9f767SBen Widawsky 6321e9f767SBen Widawsky dev_dbg(dev, "Mailbox command (opcode: %#x size: %zub) timed out\n", 6421e9f767SBen Widawsky mbox_cmd->opcode, mbox_cmd->size_in); 6521e9f767SBen Widawsky } 6621e9f767SBen Widawsky 6721e9f767SBen Widawsky /** 6821e9f767SBen Widawsky * __cxl_mem_mbox_send_cmd() - Execute a mailbox command 6921e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 7021e9f767SBen Widawsky * @mbox_cmd: Command to send to the memory device. 7121e9f767SBen Widawsky * 7221e9f767SBen Widawsky * Context: Any context. Expects mbox_mutex to be held. 7321e9f767SBen Widawsky * Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success. 7421e9f767SBen Widawsky * Caller should check the return code in @mbox_cmd to make sure it 7521e9f767SBen Widawsky * succeeded. 7621e9f767SBen Widawsky * 7721e9f767SBen Widawsky * This is a generic form of the CXL mailbox send command thus only using the 7821e9f767SBen Widawsky * registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory 7921e9f767SBen Widawsky * devices, and perhaps other types of CXL devices may have further information 8021e9f767SBen Widawsky * available upon error conditions. Driver facilities wishing to send mailbox 8121e9f767SBen Widawsky * commands should use the wrapper command. 8221e9f767SBen Widawsky * 8321e9f767SBen Widawsky * The CXL spec allows for up to two mailboxes. The intention is for the primary 8421e9f767SBen Widawsky * mailbox to be OS controlled and the secondary mailbox to be used by system 8521e9f767SBen Widawsky * firmware. This allows the OS and firmware to communicate with the device and 8621e9f767SBen Widawsky * not need to coordinate with each other. The driver only uses the primary 8721e9f767SBen Widawsky * mailbox. 8821e9f767SBen Widawsky */ 8921e9f767SBen Widawsky static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, 90b64955a9SDan Williams struct cxl_mbox_cmd *mbox_cmd) 9121e9f767SBen Widawsky { 9221e9f767SBen Widawsky void __iomem *payload = cxlm->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET; 9399e222a5SDan Williams struct device *dev = cxlm->dev; 9421e9f767SBen Widawsky u64 cmd_reg, status_reg; 9521e9f767SBen Widawsky size_t out_len; 9621e9f767SBen Widawsky int rc; 9721e9f767SBen Widawsky 9821e9f767SBen Widawsky lockdep_assert_held(&cxlm->mbox_mutex); 9921e9f767SBen Widawsky 10021e9f767SBen Widawsky /* 10121e9f767SBen Widawsky * Here are the steps from 8.2.8.4 of the CXL 2.0 spec. 10221e9f767SBen Widawsky * 1. Caller reads MB Control Register to verify doorbell is clear 10321e9f767SBen Widawsky * 2. Caller writes Command Register 10421e9f767SBen Widawsky * 3. Caller writes Command Payload Registers if input payload is non-empty 10521e9f767SBen Widawsky * 4. Caller writes MB Control Register to set doorbell 10621e9f767SBen Widawsky * 5. Caller either polls for doorbell to be clear or waits for interrupt if configured 10721e9f767SBen Widawsky * 6. Caller reads MB Status Register to fetch Return code 10821e9f767SBen Widawsky * 7. If command successful, Caller reads Command Register to get Payload Length 10921e9f767SBen Widawsky * 8. If output payload is non-empty, host reads Command Payload Registers 11021e9f767SBen Widawsky * 11121e9f767SBen Widawsky * Hardware is free to do whatever it wants before the doorbell is rung, 11221e9f767SBen Widawsky * and isn't allowed to change anything after it clears the doorbell. As 11321e9f767SBen Widawsky * such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can 11421e9f767SBen Widawsky * also happen in any order (though some orders might not make sense). 11521e9f767SBen Widawsky */ 11621e9f767SBen Widawsky 11721e9f767SBen Widawsky /* #1 */ 11821e9f767SBen Widawsky if (cxl_doorbell_busy(cxlm)) { 11999e222a5SDan Williams dev_err_ratelimited(dev, "Mailbox re-busy after acquiring\n"); 12021e9f767SBen Widawsky return -EBUSY; 12121e9f767SBen Widawsky } 12221e9f767SBen Widawsky 12321e9f767SBen Widawsky cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK, 12421e9f767SBen Widawsky mbox_cmd->opcode); 12521e9f767SBen Widawsky if (mbox_cmd->size_in) { 12621e9f767SBen Widawsky if (WARN_ON(!mbox_cmd->payload_in)) 12721e9f767SBen Widawsky return -EINVAL; 12821e9f767SBen Widawsky 12921e9f767SBen Widawsky cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, 13021e9f767SBen Widawsky mbox_cmd->size_in); 13121e9f767SBen Widawsky memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in); 13221e9f767SBen Widawsky } 13321e9f767SBen Widawsky 13421e9f767SBen Widawsky /* #2, #3 */ 13521e9f767SBen Widawsky writeq(cmd_reg, cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 13621e9f767SBen Widawsky 13721e9f767SBen Widawsky /* #4 */ 13899e222a5SDan Williams dev_dbg(dev, "Sending command\n"); 13921e9f767SBen Widawsky writel(CXLDEV_MBOX_CTRL_DOORBELL, 14021e9f767SBen Widawsky cxlm->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET); 14121e9f767SBen Widawsky 14221e9f767SBen Widawsky /* #5 */ 14321e9f767SBen Widawsky rc = cxl_mem_wait_for_doorbell(cxlm); 14421e9f767SBen Widawsky if (rc == -ETIMEDOUT) { 14521e9f767SBen Widawsky cxl_mem_mbox_timeout(cxlm, mbox_cmd); 14621e9f767SBen Widawsky return rc; 14721e9f767SBen Widawsky } 14821e9f767SBen Widawsky 14921e9f767SBen Widawsky /* #6 */ 15021e9f767SBen Widawsky status_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET); 15121e9f767SBen Widawsky mbox_cmd->return_code = 15221e9f767SBen Widawsky FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg); 15321e9f767SBen Widawsky 15421e9f767SBen Widawsky if (mbox_cmd->return_code != 0) { 15599e222a5SDan Williams dev_dbg(dev, "Mailbox operation had an error\n"); 15621e9f767SBen Widawsky return 0; 15721e9f767SBen Widawsky } 15821e9f767SBen Widawsky 15921e9f767SBen Widawsky /* #7 */ 16021e9f767SBen Widawsky cmd_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 16121e9f767SBen Widawsky out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg); 16221e9f767SBen Widawsky 16321e9f767SBen Widawsky /* #8 */ 16421e9f767SBen Widawsky if (out_len && mbox_cmd->payload_out) { 16521e9f767SBen Widawsky /* 16621e9f767SBen Widawsky * Sanitize the copy. If hardware misbehaves, out_len per the 16721e9f767SBen Widawsky * spec can actually be greater than the max allowed size (21 16821e9f767SBen Widawsky * bits available but spec defined 1M max). The caller also may 16921e9f767SBen Widawsky * have requested less data than the hardware supplied even 17021e9f767SBen Widawsky * within spec. 17121e9f767SBen Widawsky */ 17221e9f767SBen Widawsky size_t n = min3(mbox_cmd->size_out, cxlm->payload_size, out_len); 17321e9f767SBen Widawsky 17421e9f767SBen Widawsky memcpy_fromio(mbox_cmd->payload_out, payload, n); 17521e9f767SBen Widawsky mbox_cmd->size_out = n; 17621e9f767SBen Widawsky } else { 17721e9f767SBen Widawsky mbox_cmd->size_out = 0; 17821e9f767SBen Widawsky } 17921e9f767SBen Widawsky 18021e9f767SBen Widawsky return 0; 18121e9f767SBen Widawsky } 18221e9f767SBen Widawsky 18321e9f767SBen Widawsky /** 18421e9f767SBen Widawsky * cxl_mem_mbox_get() - Acquire exclusive access to the mailbox. 18521e9f767SBen Widawsky * @cxlm: The memory device to gain access to. 18621e9f767SBen Widawsky * 18721e9f767SBen Widawsky * Context: Any context. Takes the mbox_mutex. 18821e9f767SBen Widawsky * Return: 0 if exclusive access was acquired. 18921e9f767SBen Widawsky */ 19021e9f767SBen Widawsky static int cxl_mem_mbox_get(struct cxl_mem *cxlm) 19121e9f767SBen Widawsky { 19299e222a5SDan Williams struct device *dev = cxlm->dev; 19321e9f767SBen Widawsky u64 md_status; 19421e9f767SBen Widawsky int rc; 19521e9f767SBen Widawsky 19621e9f767SBen Widawsky mutex_lock_io(&cxlm->mbox_mutex); 19721e9f767SBen Widawsky 19821e9f767SBen Widawsky /* 19921e9f767SBen Widawsky * XXX: There is some amount of ambiguity in the 2.0 version of the spec 20021e9f767SBen Widawsky * around the mailbox interface ready (8.2.8.5.1.1). The purpose of the 20121e9f767SBen Widawsky * bit is to allow firmware running on the device to notify the driver 20221e9f767SBen Widawsky * that it's ready to receive commands. It is unclear if the bit needs 20321e9f767SBen Widawsky * to be read for each transaction mailbox, ie. the firmware can switch 20421e9f767SBen Widawsky * it on and off as needed. Second, there is no defined timeout for 20521e9f767SBen Widawsky * mailbox ready, like there is for the doorbell interface. 20621e9f767SBen Widawsky * 20721e9f767SBen Widawsky * Assumptions: 20821e9f767SBen Widawsky * 1. The firmware might toggle the Mailbox Interface Ready bit, check 20921e9f767SBen Widawsky * it for every command. 21021e9f767SBen Widawsky * 21121e9f767SBen Widawsky * 2. If the doorbell is clear, the firmware should have first set the 21221e9f767SBen Widawsky * Mailbox Interface Ready bit. Therefore, waiting for the doorbell 21321e9f767SBen Widawsky * to be ready is sufficient. 21421e9f767SBen Widawsky */ 21521e9f767SBen Widawsky rc = cxl_mem_wait_for_doorbell(cxlm); 21621e9f767SBen Widawsky if (rc) { 21721e9f767SBen Widawsky dev_warn(dev, "Mailbox interface not ready\n"); 21821e9f767SBen Widawsky goto out; 21921e9f767SBen Widawsky } 22021e9f767SBen Widawsky 22121e9f767SBen Widawsky md_status = readq(cxlm->regs.memdev + CXLMDEV_STATUS_OFFSET); 22221e9f767SBen Widawsky if (!(md_status & CXLMDEV_MBOX_IF_READY && CXLMDEV_READY(md_status))) { 22321e9f767SBen Widawsky dev_err(dev, "mbox: reported doorbell ready, but not mbox ready\n"); 22421e9f767SBen Widawsky rc = -EBUSY; 22521e9f767SBen Widawsky goto out; 22621e9f767SBen Widawsky } 22721e9f767SBen Widawsky 22821e9f767SBen Widawsky /* 22921e9f767SBen Widawsky * Hardware shouldn't allow a ready status but also have failure bits 23021e9f767SBen Widawsky * set. Spit out an error, this should be a bug report 23121e9f767SBen Widawsky */ 23221e9f767SBen Widawsky rc = -EFAULT; 23321e9f767SBen Widawsky if (md_status & CXLMDEV_DEV_FATAL) { 23421e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but fatal\n"); 23521e9f767SBen Widawsky goto out; 23621e9f767SBen Widawsky } 23721e9f767SBen Widawsky if (md_status & CXLMDEV_FW_HALT) { 23821e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but halted\n"); 23921e9f767SBen Widawsky goto out; 24021e9f767SBen Widawsky } 24121e9f767SBen Widawsky if (CXLMDEV_RESET_NEEDED(md_status)) { 24221e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but reset needed\n"); 24321e9f767SBen Widawsky goto out; 24421e9f767SBen Widawsky } 24521e9f767SBen Widawsky 24621e9f767SBen Widawsky /* with lock held */ 24721e9f767SBen Widawsky return 0; 24821e9f767SBen Widawsky 24921e9f767SBen Widawsky out: 25021e9f767SBen Widawsky mutex_unlock(&cxlm->mbox_mutex); 25121e9f767SBen Widawsky return rc; 25221e9f767SBen Widawsky } 25321e9f767SBen Widawsky 25421e9f767SBen Widawsky /** 25521e9f767SBen Widawsky * cxl_mem_mbox_put() - Release exclusive access to the mailbox. 25621e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 25721e9f767SBen Widawsky * 25821e9f767SBen Widawsky * Context: Any context. Expects mbox_mutex to be held. 25921e9f767SBen Widawsky */ 26021e9f767SBen Widawsky static void cxl_mem_mbox_put(struct cxl_mem *cxlm) 26121e9f767SBen Widawsky { 26221e9f767SBen Widawsky mutex_unlock(&cxlm->mbox_mutex); 26321e9f767SBen Widawsky } 26421e9f767SBen Widawsky 265b64955a9SDan Williams static int cxl_pci_mbox_send(struct cxl_mem *cxlm, struct cxl_mbox_cmd *cmd) 266b64955a9SDan Williams { 267b64955a9SDan Williams int rc; 268b64955a9SDan Williams 269b64955a9SDan Williams rc = cxl_mem_mbox_get(cxlm); 270b64955a9SDan Williams if (rc) 271b64955a9SDan Williams return rc; 272b64955a9SDan Williams 273b64955a9SDan Williams rc = __cxl_mem_mbox_send_cmd(cxlm, cmd); 274b64955a9SDan Williams cxl_mem_mbox_put(cxlm); 275b64955a9SDan Williams 276b64955a9SDan Williams return rc; 277b64955a9SDan Williams } 278b64955a9SDan Williams 27921e9f767SBen Widawsky static int cxl_mem_setup_mailbox(struct cxl_mem *cxlm) 28021e9f767SBen Widawsky { 28121e9f767SBen Widawsky const int cap = readl(cxlm->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET); 28221e9f767SBen Widawsky 283b64955a9SDan Williams cxlm->mbox_send = cxl_pci_mbox_send; 28421e9f767SBen Widawsky cxlm->payload_size = 28521e9f767SBen Widawsky 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap); 28621e9f767SBen Widawsky 28721e9f767SBen Widawsky /* 28821e9f767SBen Widawsky * CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register 28921e9f767SBen Widawsky * 29021e9f767SBen Widawsky * If the size is too small, mandatory commands will not work and so 29121e9f767SBen Widawsky * there's no point in going forward. If the size is too large, there's 29221e9f767SBen Widawsky * no harm is soft limiting it. 29321e9f767SBen Widawsky */ 29421e9f767SBen Widawsky cxlm->payload_size = min_t(size_t, cxlm->payload_size, SZ_1M); 29521e9f767SBen Widawsky if (cxlm->payload_size < 256) { 29699e222a5SDan Williams dev_err(cxlm->dev, "Mailbox is too small (%zub)", 29721e9f767SBen Widawsky cxlm->payload_size); 29821e9f767SBen Widawsky return -ENXIO; 29921e9f767SBen Widawsky } 30021e9f767SBen Widawsky 30199e222a5SDan Williams dev_dbg(cxlm->dev, "Mailbox payload sized %zu", 30221e9f767SBen Widawsky cxlm->payload_size); 30321e9f767SBen Widawsky 30421e9f767SBen Widawsky return 0; 30521e9f767SBen Widawsky } 30621e9f767SBen Widawsky 30707d62eacSIra Weiny static void __iomem *cxl_mem_map_regblock(struct cxl_mem *cxlm, 30807d62eacSIra Weiny u8 bar, u64 offset) 3091b0a1a2aSBen Widawsky { 310f8a7e8c2SIra Weiny void __iomem *addr; 31199e222a5SDan Williams struct device *dev = cxlm->dev; 31299e222a5SDan Williams struct pci_dev *pdev = to_pci_dev(dev); 3131b0a1a2aSBen Widawsky 31421e9f767SBen Widawsky /* Basic sanity check that BAR is big enough */ 31521e9f767SBen Widawsky if (pci_resource_len(pdev, bar) < offset) { 31621e9f767SBen Widawsky dev_err(dev, "BAR%d: %pr: too small (offset: %#llx)\n", bar, 31721e9f767SBen Widawsky &pdev->resource[bar], (unsigned long long)offset); 3186630d31cSBen Widawsky return IOMEM_ERR_PTR(-ENXIO); 31921e9f767SBen Widawsky } 32021e9f767SBen Widawsky 32130af9729SIra Weiny addr = pci_iomap(pdev, bar, 0); 322f8a7e8c2SIra Weiny if (!addr) { 32321e9f767SBen Widawsky dev_err(dev, "failed to map registers\n"); 324f8a7e8c2SIra Weiny return addr; 32521e9f767SBen Widawsky } 32621e9f767SBen Widawsky 327f8a7e8c2SIra Weiny dev_dbg(dev, "Mapped CXL Memory Device resource bar %u @ %#llx\n", 328f8a7e8c2SIra Weiny bar, offset); 3296630d31cSBen Widawsky 33030af9729SIra Weiny return addr; 33130af9729SIra Weiny } 33230af9729SIra Weiny 33330af9729SIra Weiny static void cxl_mem_unmap_regblock(struct cxl_mem *cxlm, void __iomem *base) 33430af9729SIra Weiny { 33599e222a5SDan Williams pci_iounmap(to_pci_dev(cxlm->dev), base); 33621e9f767SBen Widawsky } 33721e9f767SBen Widawsky 33821e9f767SBen Widawsky static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec) 33921e9f767SBen Widawsky { 34021e9f767SBen Widawsky int pos; 34121e9f767SBen Widawsky 34221e9f767SBen Widawsky pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DVSEC); 34321e9f767SBen Widawsky if (!pos) 34421e9f767SBen Widawsky return 0; 34521e9f767SBen Widawsky 34621e9f767SBen Widawsky while (pos) { 34721e9f767SBen Widawsky u16 vendor, id; 34821e9f767SBen Widawsky 34921e9f767SBen Widawsky pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER1, &vendor); 35021e9f767SBen Widawsky pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER2, &id); 35121e9f767SBen Widawsky if (vendor == PCI_DVSEC_VENDOR_ID_CXL && dvsec == id) 35221e9f767SBen Widawsky return pos; 35321e9f767SBen Widawsky 35421e9f767SBen Widawsky pos = pci_find_next_ext_capability(pdev, pos, 35521e9f767SBen Widawsky PCI_EXT_CAP_ID_DVSEC); 35621e9f767SBen Widawsky } 35721e9f767SBen Widawsky 35821e9f767SBen Widawsky return 0; 35921e9f767SBen Widawsky } 36021e9f767SBen Widawsky 36130af9729SIra Weiny static int cxl_probe_regs(struct cxl_mem *cxlm, void __iomem *base, 36230af9729SIra Weiny struct cxl_register_map *map) 36330af9729SIra Weiny { 36408422378SBen Widawsky struct cxl_component_reg_map *comp_map; 36530af9729SIra Weiny struct cxl_device_reg_map *dev_map; 36699e222a5SDan Williams struct device *dev = cxlm->dev; 36730af9729SIra Weiny 36830af9729SIra Weiny switch (map->reg_type) { 36908422378SBen Widawsky case CXL_REGLOC_RBI_COMPONENT: 37008422378SBen Widawsky comp_map = &map->component_map; 37108422378SBen Widawsky cxl_probe_component_regs(dev, base, comp_map); 37208422378SBen Widawsky if (!comp_map->hdm_decoder.valid) { 37308422378SBen Widawsky dev_err(dev, "HDM decoder registers not found\n"); 37408422378SBen Widawsky return -ENXIO; 37508422378SBen Widawsky } 37608422378SBen Widawsky 37708422378SBen Widawsky dev_dbg(dev, "Set up component registers\n"); 37808422378SBen Widawsky break; 37930af9729SIra Weiny case CXL_REGLOC_RBI_MEMDEV: 38030af9729SIra Weiny dev_map = &map->device_map; 38130af9729SIra Weiny cxl_probe_device_regs(dev, base, dev_map); 38230af9729SIra Weiny if (!dev_map->status.valid || !dev_map->mbox.valid || 38330af9729SIra Weiny !dev_map->memdev.valid) { 38430af9729SIra Weiny dev_err(dev, "registers not found: %s%s%s\n", 38530af9729SIra Weiny !dev_map->status.valid ? "status " : "", 386da582aa5SLi Qiang (Johnny Li) !dev_map->mbox.valid ? "mbox " : "", 387da582aa5SLi Qiang (Johnny Li) !dev_map->memdev.valid ? "memdev " : ""); 38830af9729SIra Weiny return -ENXIO; 38930af9729SIra Weiny } 39030af9729SIra Weiny 39130af9729SIra Weiny dev_dbg(dev, "Probing device registers...\n"); 39230af9729SIra Weiny break; 39330af9729SIra Weiny default: 39430af9729SIra Weiny break; 39530af9729SIra Weiny } 39630af9729SIra Weiny 39730af9729SIra Weiny return 0; 39830af9729SIra Weiny } 39930af9729SIra Weiny 40030af9729SIra Weiny static int cxl_map_regs(struct cxl_mem *cxlm, struct cxl_register_map *map) 40130af9729SIra Weiny { 40299e222a5SDan Williams struct device *dev = cxlm->dev; 40399e222a5SDan Williams struct pci_dev *pdev = to_pci_dev(dev); 40430af9729SIra Weiny 40530af9729SIra Weiny switch (map->reg_type) { 40608422378SBen Widawsky case CXL_REGLOC_RBI_COMPONENT: 40708422378SBen Widawsky cxl_map_component_regs(pdev, &cxlm->regs.component, map); 40808422378SBen Widawsky dev_dbg(dev, "Mapping component registers...\n"); 40908422378SBen Widawsky break; 41030af9729SIra Weiny case CXL_REGLOC_RBI_MEMDEV: 41130af9729SIra Weiny cxl_map_device_regs(pdev, &cxlm->regs.device_regs, map); 41230af9729SIra Weiny dev_dbg(dev, "Probing device registers...\n"); 41330af9729SIra Weiny break; 41430af9729SIra Weiny default: 41530af9729SIra Weiny break; 41630af9729SIra Weiny } 41730af9729SIra Weiny 41830af9729SIra Weiny return 0; 41930af9729SIra Weiny } 42030af9729SIra Weiny 42107d62eacSIra Weiny static void cxl_decode_register_block(u32 reg_lo, u32 reg_hi, 42207d62eacSIra Weiny u8 *bar, u64 *offset, u8 *reg_type) 42307d62eacSIra Weiny { 42407d62eacSIra Weiny *offset = ((u64)reg_hi << 32) | (reg_lo & CXL_REGLOC_ADDR_MASK); 42507d62eacSIra Weiny *bar = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo); 42607d62eacSIra Weiny *reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); 42707d62eacSIra Weiny } 42807d62eacSIra Weiny 4291d5a4159SBen Widawsky /** 4301d5a4159SBen Widawsky * cxl_mem_setup_regs() - Setup necessary MMIO. 4311d5a4159SBen Widawsky * @cxlm: The CXL memory device to communicate with. 4321d5a4159SBen Widawsky * 4331d5a4159SBen Widawsky * Return: 0 if all necessary registers mapped. 4341d5a4159SBen Widawsky * 4351d5a4159SBen Widawsky * A memory device is required by spec to implement a certain set of MMIO 4361d5a4159SBen Widawsky * regions. The purpose of this function is to enumerate and map those 4371d5a4159SBen Widawsky * registers. 4381d5a4159SBen Widawsky */ 4391d5a4159SBen Widawsky static int cxl_mem_setup_regs(struct cxl_mem *cxlm) 4401d5a4159SBen Widawsky { 4416630d31cSBen Widawsky void __iomem *base; 44299e222a5SDan Williams u32 regloc_size, regblocks; 44399e222a5SDan Williams int regloc, i, n_maps, ret = 0; 44499e222a5SDan Williams struct device *dev = cxlm->dev; 44599e222a5SDan Williams struct pci_dev *pdev = to_pci_dev(dev); 4465b68705dSBen Widawsky struct cxl_register_map *map, maps[CXL_REGLOC_RBI_TYPES]; 4471d5a4159SBen Widawsky 4484ad6181eSBen Widawsky regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_DVSEC_ID); 4491d5a4159SBen Widawsky if (!regloc) { 4501d5a4159SBen Widawsky dev_err(dev, "register location dvsec not found\n"); 4511d5a4159SBen Widawsky return -ENXIO; 4521d5a4159SBen Widawsky } 4531d5a4159SBen Widawsky 454f8a7e8c2SIra Weiny if (pci_request_mem_regions(pdev, pci_name(pdev))) 455f8a7e8c2SIra Weiny return -ENODEV; 456f8a7e8c2SIra Weiny 4571d5a4159SBen Widawsky /* Get the size of the Register Locator DVSEC */ 4581d5a4159SBen Widawsky pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size); 4591d5a4159SBen Widawsky regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size); 4601d5a4159SBen Widawsky 4611d5a4159SBen Widawsky regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET; 4621d5a4159SBen Widawsky regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8; 4631d5a4159SBen Widawsky 4645b68705dSBen Widawsky for (i = 0, n_maps = 0; i < regblocks; i++, regloc += 8) { 4651d5a4159SBen Widawsky u32 reg_lo, reg_hi; 4661d5a4159SBen Widawsky u8 reg_type; 46707d62eacSIra Weiny u64 offset; 46807d62eacSIra Weiny u8 bar; 4691d5a4159SBen Widawsky 4701d5a4159SBen Widawsky pci_read_config_dword(pdev, regloc, ®_lo); 4711d5a4159SBen Widawsky pci_read_config_dword(pdev, regloc + 4, ®_hi); 4721d5a4159SBen Widawsky 47307d62eacSIra Weiny cxl_decode_register_block(reg_lo, reg_hi, &bar, &offset, 47407d62eacSIra Weiny ®_type); 47507d62eacSIra Weiny 47607d62eacSIra Weiny dev_dbg(dev, "Found register block in bar %u @ 0x%llx of type %u\n", 47707d62eacSIra Weiny bar, offset, reg_type); 4781d5a4159SBen Widawsky 4791e39db57SBen Widawsky /* Ignore unknown register block types */ 4801e39db57SBen Widawsky if (reg_type > CXL_REGLOC_RBI_MEMDEV) 4811e39db57SBen Widawsky continue; 4821e39db57SBen Widawsky 48307d62eacSIra Weiny base = cxl_mem_map_regblock(cxlm, bar, offset); 4845b68705dSBen Widawsky if (!base) 4855b68705dSBen Widawsky return -ENOMEM; 4861d5a4159SBen Widawsky 4875b68705dSBen Widawsky map = &maps[n_maps]; 48830af9729SIra Weiny map->barno = bar; 48930af9729SIra Weiny map->block_offset = offset; 49030af9729SIra Weiny map->reg_type = reg_type; 49130af9729SIra Weiny 49230af9729SIra Weiny ret = cxl_probe_regs(cxlm, base + offset, map); 49330af9729SIra Weiny 49430af9729SIra Weiny /* Always unmap the regblock regardless of probe success */ 49530af9729SIra Weiny cxl_mem_unmap_regblock(cxlm, base); 49630af9729SIra Weiny 49730af9729SIra Weiny if (ret) 4985b68705dSBen Widawsky return ret; 4995b68705dSBen Widawsky 5005b68705dSBen Widawsky n_maps++; 5011d5a4159SBen Widawsky } 5021d5a4159SBen Widawsky 5039a016527SIra Weiny pci_release_mem_regions(pdev); 5049a016527SIra Weiny 5055b68705dSBen Widawsky for (i = 0; i < n_maps; i++) { 5065b68705dSBen Widawsky ret = cxl_map_regs(cxlm, &maps[i]); 50730af9729SIra Weiny if (ret) 5085b68705dSBen Widawsky break; 50930af9729SIra Weiny } 51030af9729SIra Weiny 51130af9729SIra Weiny return ret; 5121d5a4159SBen Widawsky } 5131d5a4159SBen Widawsky 51421e9f767SBen Widawsky static int cxl_mem_probe(struct pci_dev *pdev, const struct pci_device_id *id) 51521e9f767SBen Widawsky { 51621083f51SDan Williams struct cxl_memdev *cxlmd; 5171b0a1a2aSBen Widawsky struct cxl_mem *cxlm; 5181d5a4159SBen Widawsky int rc; 51921e9f767SBen Widawsky 520*5a2328f4SDan Williams /* 521*5a2328f4SDan Williams * Double check the anonymous union trickery in struct cxl_regs 522*5a2328f4SDan Williams * FIXME switch to struct_group() 523*5a2328f4SDan Williams */ 524*5a2328f4SDan Williams BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) != 525*5a2328f4SDan Williams offsetof(struct cxl_regs, device_regs.memdev)); 526*5a2328f4SDan Williams 52721e9f767SBen Widawsky rc = pcim_enable_device(pdev); 52821e9f767SBen Widawsky if (rc) 52921e9f767SBen Widawsky return rc; 53021e9f767SBen Widawsky 53199e222a5SDan Williams cxlm = cxl_mem_create(&pdev->dev); 5321b0a1a2aSBen Widawsky if (IS_ERR(cxlm)) 5331b0a1a2aSBen Widawsky return PTR_ERR(cxlm); 5341b0a1a2aSBen Widawsky 53521e9f767SBen Widawsky rc = cxl_mem_setup_regs(cxlm); 53621e9f767SBen Widawsky if (rc) 53721e9f767SBen Widawsky return rc; 53821e9f767SBen Widawsky 53921e9f767SBen Widawsky rc = cxl_mem_setup_mailbox(cxlm); 54021e9f767SBen Widawsky if (rc) 54121e9f767SBen Widawsky return rc; 54221e9f767SBen Widawsky 54321e9f767SBen Widawsky rc = cxl_mem_enumerate_cmds(cxlm); 54421e9f767SBen Widawsky if (rc) 54521e9f767SBen Widawsky return rc; 54621e9f767SBen Widawsky 54721e9f767SBen Widawsky rc = cxl_mem_identify(cxlm); 54821e9f767SBen Widawsky if (rc) 54921e9f767SBen Widawsky return rc; 55021e9f767SBen Widawsky 551f847502aSIra Weiny rc = cxl_mem_create_range_info(cxlm); 552f847502aSIra Weiny if (rc) 553f847502aSIra Weiny return rc; 554f847502aSIra Weiny 5554faf31b4SDan Williams cxlmd = devm_cxl_add_memdev(cxlm); 55621083f51SDan Williams if (IS_ERR(cxlmd)) 55721083f51SDan Williams return PTR_ERR(cxlmd); 55821083f51SDan Williams 55921083f51SDan Williams if (range_len(&cxlm->pmem_range) && IS_ENABLED(CONFIG_CXL_PMEM)) 56021083f51SDan Williams rc = devm_cxl_add_nvdimm(&pdev->dev, cxlmd); 56121083f51SDan Williams 56221083f51SDan Williams return rc; 56321e9f767SBen Widawsky } 56421e9f767SBen Widawsky 56521e9f767SBen Widawsky static const struct pci_device_id cxl_mem_pci_tbl[] = { 56621e9f767SBen Widawsky /* PCI class code for CXL.mem Type-3 Devices */ 56721e9f767SBen Widawsky { PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)}, 56821e9f767SBen Widawsky { /* terminate list */ }, 56921e9f767SBen Widawsky }; 57021e9f767SBen Widawsky MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl); 57121e9f767SBen Widawsky 57221e9f767SBen Widawsky static struct pci_driver cxl_mem_driver = { 57321e9f767SBen Widawsky .name = KBUILD_MODNAME, 57421e9f767SBen Widawsky .id_table = cxl_mem_pci_tbl, 57521e9f767SBen Widawsky .probe = cxl_mem_probe, 57621e9f767SBen Widawsky .driver = { 57721e9f767SBen Widawsky .probe_type = PROBE_PREFER_ASYNCHRONOUS, 57821e9f767SBen Widawsky }, 57921e9f767SBen Widawsky }; 58021e9f767SBen Widawsky 58121e9f767SBen Widawsky MODULE_LICENSE("GPL v2"); 582*5a2328f4SDan Williams module_pci_driver(cxl_mem_driver); 58321e9f767SBen Widawsky MODULE_IMPORT_NS(CXL); 584