121e9f767SBen Widawsky // SPDX-License-Identifier: GPL-2.0-only 221e9f767SBen Widawsky /* Copyright(c) 2020 Intel Corporation. All rights reserved. */ 34faf31b4SDan Williams #include <linux/io-64-nonatomic-lo-hi.h> 421e9f767SBen Widawsky #include <linux/module.h> 521e9f767SBen Widawsky #include <linux/sizes.h> 621e9f767SBen Widawsky #include <linux/mutex.h> 730af9729SIra Weiny #include <linux/list.h> 821e9f767SBen Widawsky #include <linux/pci.h> 921e9f767SBen Widawsky #include <linux/io.h> 105161a55cSBen Widawsky #include "cxlmem.h" 1121e9f767SBen Widawsky #include "pci.h" 1221e9f767SBen Widawsky #include "cxl.h" 1321e9f767SBen Widawsky 1421e9f767SBen Widawsky /** 1521e9f767SBen Widawsky * DOC: cxl pci 1621e9f767SBen Widawsky * 1721e9f767SBen Widawsky * This implements the PCI exclusive functionality for a CXL device as it is 1821e9f767SBen Widawsky * defined by the Compute Express Link specification. CXL devices may surface 19ed97afb5SBen Widawsky * certain functionality even if it isn't CXL enabled. While this driver is 20ed97afb5SBen Widawsky * focused around the PCI specific aspects of a CXL device, it binds to the 21ed97afb5SBen Widawsky * specific CXL memory device class code, and therefore the implementation of 22ed97afb5SBen Widawsky * cxl_pci is focused around CXL memory devices. 2321e9f767SBen Widawsky * 2421e9f767SBen Widawsky * The driver has several responsibilities, mainly: 2521e9f767SBen Widawsky * - Create the memX device and register on the CXL bus. 2621e9f767SBen Widawsky * - Enumerate device's register interface and map them. 27ed97afb5SBen Widawsky * - Registers nvdimm bridge device with cxl_core. 28ed97afb5SBen Widawsky * - Registers a CXL mailbox with cxl_core. 2921e9f767SBen Widawsky */ 3021e9f767SBen Widawsky 3121e9f767SBen Widawsky #define cxl_doorbell_busy(cxlm) \ 3221e9f767SBen Widawsky (readl((cxlm)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \ 3321e9f767SBen Widawsky CXLDEV_MBOX_CTRL_DOORBELL) 3421e9f767SBen Widawsky 3521e9f767SBen Widawsky /* CXL 2.0 - 8.2.8.4 */ 3621e9f767SBen Widawsky #define CXL_MAILBOX_TIMEOUT_MS (2 * HZ) 3721e9f767SBen Widawsky 38ed97afb5SBen Widawsky static int cxl_pci_mbox_wait_for_doorbell(struct cxl_mem *cxlm) 3921e9f767SBen Widawsky { 4021e9f767SBen Widawsky const unsigned long start = jiffies; 4121e9f767SBen Widawsky unsigned long end = start; 4221e9f767SBen Widawsky 4321e9f767SBen Widawsky while (cxl_doorbell_busy(cxlm)) { 4421e9f767SBen Widawsky end = jiffies; 4521e9f767SBen Widawsky 4621e9f767SBen Widawsky if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) { 4721e9f767SBen Widawsky /* Check again in case preempted before timeout test */ 4821e9f767SBen Widawsky if (!cxl_doorbell_busy(cxlm)) 4921e9f767SBen Widawsky break; 5021e9f767SBen Widawsky return -ETIMEDOUT; 5121e9f767SBen Widawsky } 5221e9f767SBen Widawsky cpu_relax(); 5321e9f767SBen Widawsky } 5421e9f767SBen Widawsky 5599e222a5SDan Williams dev_dbg(cxlm->dev, "Doorbell wait took %dms", 5621e9f767SBen Widawsky jiffies_to_msecs(end) - jiffies_to_msecs(start)); 5721e9f767SBen Widawsky return 0; 5821e9f767SBen Widawsky } 5921e9f767SBen Widawsky 60ed97afb5SBen Widawsky static void cxl_pci_mbox_timeout(struct cxl_mem *cxlm, 61b64955a9SDan Williams struct cxl_mbox_cmd *mbox_cmd) 6221e9f767SBen Widawsky { 6399e222a5SDan Williams struct device *dev = cxlm->dev; 6421e9f767SBen Widawsky 6521e9f767SBen Widawsky dev_dbg(dev, "Mailbox command (opcode: %#x size: %zub) timed out\n", 6621e9f767SBen Widawsky mbox_cmd->opcode, mbox_cmd->size_in); 6721e9f767SBen Widawsky } 6821e9f767SBen Widawsky 6921e9f767SBen Widawsky /** 70ed97afb5SBen Widawsky * __cxl_pci_mbox_send_cmd() - Execute a mailbox command 7121e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 7221e9f767SBen Widawsky * @mbox_cmd: Command to send to the memory device. 7321e9f767SBen Widawsky * 7421e9f767SBen Widawsky * Context: Any context. Expects mbox_mutex to be held. 7521e9f767SBen Widawsky * Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success. 7621e9f767SBen Widawsky * Caller should check the return code in @mbox_cmd to make sure it 7721e9f767SBen Widawsky * succeeded. 7821e9f767SBen Widawsky * 7921e9f767SBen Widawsky * This is a generic form of the CXL mailbox send command thus only using the 8021e9f767SBen Widawsky * registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory 8121e9f767SBen Widawsky * devices, and perhaps other types of CXL devices may have further information 8221e9f767SBen Widawsky * available upon error conditions. Driver facilities wishing to send mailbox 8321e9f767SBen Widawsky * commands should use the wrapper command. 8421e9f767SBen Widawsky * 8521e9f767SBen Widawsky * The CXL spec allows for up to two mailboxes. The intention is for the primary 8621e9f767SBen Widawsky * mailbox to be OS controlled and the secondary mailbox to be used by system 8721e9f767SBen Widawsky * firmware. This allows the OS and firmware to communicate with the device and 8821e9f767SBen Widawsky * not need to coordinate with each other. The driver only uses the primary 8921e9f767SBen Widawsky * mailbox. 9021e9f767SBen Widawsky */ 91ed97afb5SBen Widawsky static int __cxl_pci_mbox_send_cmd(struct cxl_mem *cxlm, 92b64955a9SDan Williams struct cxl_mbox_cmd *mbox_cmd) 9321e9f767SBen Widawsky { 9421e9f767SBen Widawsky void __iomem *payload = cxlm->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET; 9599e222a5SDan Williams struct device *dev = cxlm->dev; 9621e9f767SBen Widawsky u64 cmd_reg, status_reg; 9721e9f767SBen Widawsky size_t out_len; 9821e9f767SBen Widawsky int rc; 9921e9f767SBen Widawsky 10021e9f767SBen Widawsky lockdep_assert_held(&cxlm->mbox_mutex); 10121e9f767SBen Widawsky 10221e9f767SBen Widawsky /* 10321e9f767SBen Widawsky * Here are the steps from 8.2.8.4 of the CXL 2.0 spec. 10421e9f767SBen Widawsky * 1. Caller reads MB Control Register to verify doorbell is clear 10521e9f767SBen Widawsky * 2. Caller writes Command Register 10621e9f767SBen Widawsky * 3. Caller writes Command Payload Registers if input payload is non-empty 10721e9f767SBen Widawsky * 4. Caller writes MB Control Register to set doorbell 10821e9f767SBen Widawsky * 5. Caller either polls for doorbell to be clear or waits for interrupt if configured 10921e9f767SBen Widawsky * 6. Caller reads MB Status Register to fetch Return code 11021e9f767SBen Widawsky * 7. If command successful, Caller reads Command Register to get Payload Length 11121e9f767SBen Widawsky * 8. If output payload is non-empty, host reads Command Payload Registers 11221e9f767SBen Widawsky * 11321e9f767SBen Widawsky * Hardware is free to do whatever it wants before the doorbell is rung, 11421e9f767SBen Widawsky * and isn't allowed to change anything after it clears the doorbell. As 11521e9f767SBen Widawsky * such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can 11621e9f767SBen Widawsky * also happen in any order (though some orders might not make sense). 11721e9f767SBen Widawsky */ 11821e9f767SBen Widawsky 11921e9f767SBen Widawsky /* #1 */ 12021e9f767SBen Widawsky if (cxl_doorbell_busy(cxlm)) { 12199e222a5SDan Williams dev_err_ratelimited(dev, "Mailbox re-busy after acquiring\n"); 12221e9f767SBen Widawsky return -EBUSY; 12321e9f767SBen Widawsky } 12421e9f767SBen Widawsky 12521e9f767SBen Widawsky cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK, 12621e9f767SBen Widawsky mbox_cmd->opcode); 12721e9f767SBen Widawsky if (mbox_cmd->size_in) { 12821e9f767SBen Widawsky if (WARN_ON(!mbox_cmd->payload_in)) 12921e9f767SBen Widawsky return -EINVAL; 13021e9f767SBen Widawsky 13121e9f767SBen Widawsky cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, 13221e9f767SBen Widawsky mbox_cmd->size_in); 13321e9f767SBen Widawsky memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in); 13421e9f767SBen Widawsky } 13521e9f767SBen Widawsky 13621e9f767SBen Widawsky /* #2, #3 */ 13721e9f767SBen Widawsky writeq(cmd_reg, cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 13821e9f767SBen Widawsky 13921e9f767SBen Widawsky /* #4 */ 14099e222a5SDan Williams dev_dbg(dev, "Sending command\n"); 14121e9f767SBen Widawsky writel(CXLDEV_MBOX_CTRL_DOORBELL, 14221e9f767SBen Widawsky cxlm->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET); 14321e9f767SBen Widawsky 14421e9f767SBen Widawsky /* #5 */ 145ed97afb5SBen Widawsky rc = cxl_pci_mbox_wait_for_doorbell(cxlm); 14621e9f767SBen Widawsky if (rc == -ETIMEDOUT) { 147ed97afb5SBen Widawsky cxl_pci_mbox_timeout(cxlm, mbox_cmd); 14821e9f767SBen Widawsky return rc; 14921e9f767SBen Widawsky } 15021e9f767SBen Widawsky 15121e9f767SBen Widawsky /* #6 */ 15221e9f767SBen Widawsky status_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET); 15321e9f767SBen Widawsky mbox_cmd->return_code = 15421e9f767SBen Widawsky FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg); 15521e9f767SBen Widawsky 15621e9f767SBen Widawsky if (mbox_cmd->return_code != 0) { 15799e222a5SDan Williams dev_dbg(dev, "Mailbox operation had an error\n"); 15821e9f767SBen Widawsky return 0; 15921e9f767SBen Widawsky } 16021e9f767SBen Widawsky 16121e9f767SBen Widawsky /* #7 */ 16221e9f767SBen Widawsky cmd_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 16321e9f767SBen Widawsky out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg); 16421e9f767SBen Widawsky 16521e9f767SBen Widawsky /* #8 */ 16621e9f767SBen Widawsky if (out_len && mbox_cmd->payload_out) { 16721e9f767SBen Widawsky /* 16821e9f767SBen Widawsky * Sanitize the copy. If hardware misbehaves, out_len per the 16921e9f767SBen Widawsky * spec can actually be greater than the max allowed size (21 17021e9f767SBen Widawsky * bits available but spec defined 1M max). The caller also may 17121e9f767SBen Widawsky * have requested less data than the hardware supplied even 17221e9f767SBen Widawsky * within spec. 17321e9f767SBen Widawsky */ 17421e9f767SBen Widawsky size_t n = min3(mbox_cmd->size_out, cxlm->payload_size, out_len); 17521e9f767SBen Widawsky 17621e9f767SBen Widawsky memcpy_fromio(mbox_cmd->payload_out, payload, n); 17721e9f767SBen Widawsky mbox_cmd->size_out = n; 17821e9f767SBen Widawsky } else { 17921e9f767SBen Widawsky mbox_cmd->size_out = 0; 18021e9f767SBen Widawsky } 18121e9f767SBen Widawsky 18221e9f767SBen Widawsky return 0; 18321e9f767SBen Widawsky } 18421e9f767SBen Widawsky 18521e9f767SBen Widawsky /** 186ed97afb5SBen Widawsky * cxl_pci_mbox_get() - Acquire exclusive access to the mailbox. 18721e9f767SBen Widawsky * @cxlm: The memory device to gain access to. 18821e9f767SBen Widawsky * 18921e9f767SBen Widawsky * Context: Any context. Takes the mbox_mutex. 19021e9f767SBen Widawsky * Return: 0 if exclusive access was acquired. 19121e9f767SBen Widawsky */ 192ed97afb5SBen Widawsky static int cxl_pci_mbox_get(struct cxl_mem *cxlm) 19321e9f767SBen Widawsky { 19499e222a5SDan Williams struct device *dev = cxlm->dev; 19521e9f767SBen Widawsky u64 md_status; 19621e9f767SBen Widawsky int rc; 19721e9f767SBen Widawsky 19821e9f767SBen Widawsky mutex_lock_io(&cxlm->mbox_mutex); 19921e9f767SBen Widawsky 20021e9f767SBen Widawsky /* 20121e9f767SBen Widawsky * XXX: There is some amount of ambiguity in the 2.0 version of the spec 20221e9f767SBen Widawsky * around the mailbox interface ready (8.2.8.5.1.1). The purpose of the 20321e9f767SBen Widawsky * bit is to allow firmware running on the device to notify the driver 20421e9f767SBen Widawsky * that it's ready to receive commands. It is unclear if the bit needs 20521e9f767SBen Widawsky * to be read for each transaction mailbox, ie. the firmware can switch 20621e9f767SBen Widawsky * it on and off as needed. Second, there is no defined timeout for 20721e9f767SBen Widawsky * mailbox ready, like there is for the doorbell interface. 20821e9f767SBen Widawsky * 20921e9f767SBen Widawsky * Assumptions: 21021e9f767SBen Widawsky * 1. The firmware might toggle the Mailbox Interface Ready bit, check 21121e9f767SBen Widawsky * it for every command. 21221e9f767SBen Widawsky * 21321e9f767SBen Widawsky * 2. If the doorbell is clear, the firmware should have first set the 21421e9f767SBen Widawsky * Mailbox Interface Ready bit. Therefore, waiting for the doorbell 21521e9f767SBen Widawsky * to be ready is sufficient. 21621e9f767SBen Widawsky */ 217ed97afb5SBen Widawsky rc = cxl_pci_mbox_wait_for_doorbell(cxlm); 21821e9f767SBen Widawsky if (rc) { 21921e9f767SBen Widawsky dev_warn(dev, "Mailbox interface not ready\n"); 22021e9f767SBen Widawsky goto out; 22121e9f767SBen Widawsky } 22221e9f767SBen Widawsky 22321e9f767SBen Widawsky md_status = readq(cxlm->regs.memdev + CXLMDEV_STATUS_OFFSET); 22421e9f767SBen Widawsky if (!(md_status & CXLMDEV_MBOX_IF_READY && CXLMDEV_READY(md_status))) { 22521e9f767SBen Widawsky dev_err(dev, "mbox: reported doorbell ready, but not mbox ready\n"); 22621e9f767SBen Widawsky rc = -EBUSY; 22721e9f767SBen Widawsky goto out; 22821e9f767SBen Widawsky } 22921e9f767SBen Widawsky 23021e9f767SBen Widawsky /* 23121e9f767SBen Widawsky * Hardware shouldn't allow a ready status but also have failure bits 23221e9f767SBen Widawsky * set. Spit out an error, this should be a bug report 23321e9f767SBen Widawsky */ 23421e9f767SBen Widawsky rc = -EFAULT; 23521e9f767SBen Widawsky if (md_status & CXLMDEV_DEV_FATAL) { 23621e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but fatal\n"); 23721e9f767SBen Widawsky goto out; 23821e9f767SBen Widawsky } 23921e9f767SBen Widawsky if (md_status & CXLMDEV_FW_HALT) { 24021e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but halted\n"); 24121e9f767SBen Widawsky goto out; 24221e9f767SBen Widawsky } 24321e9f767SBen Widawsky if (CXLMDEV_RESET_NEEDED(md_status)) { 24421e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but reset needed\n"); 24521e9f767SBen Widawsky goto out; 24621e9f767SBen Widawsky } 24721e9f767SBen Widawsky 24821e9f767SBen Widawsky /* with lock held */ 24921e9f767SBen Widawsky return 0; 25021e9f767SBen Widawsky 25121e9f767SBen Widawsky out: 25221e9f767SBen Widawsky mutex_unlock(&cxlm->mbox_mutex); 25321e9f767SBen Widawsky return rc; 25421e9f767SBen Widawsky } 25521e9f767SBen Widawsky 25621e9f767SBen Widawsky /** 257ed97afb5SBen Widawsky * cxl_pci_mbox_put() - Release exclusive access to the mailbox. 25821e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 25921e9f767SBen Widawsky * 26021e9f767SBen Widawsky * Context: Any context. Expects mbox_mutex to be held. 26121e9f767SBen Widawsky */ 262ed97afb5SBen Widawsky static void cxl_pci_mbox_put(struct cxl_mem *cxlm) 26321e9f767SBen Widawsky { 26421e9f767SBen Widawsky mutex_unlock(&cxlm->mbox_mutex); 26521e9f767SBen Widawsky } 26621e9f767SBen Widawsky 267b64955a9SDan Williams static int cxl_pci_mbox_send(struct cxl_mem *cxlm, struct cxl_mbox_cmd *cmd) 268b64955a9SDan Williams { 269b64955a9SDan Williams int rc; 270b64955a9SDan Williams 271ed97afb5SBen Widawsky rc = cxl_pci_mbox_get(cxlm); 272b64955a9SDan Williams if (rc) 273b64955a9SDan Williams return rc; 274b64955a9SDan Williams 275ed97afb5SBen Widawsky rc = __cxl_pci_mbox_send_cmd(cxlm, cmd); 276ed97afb5SBen Widawsky cxl_pci_mbox_put(cxlm); 277b64955a9SDan Williams 278b64955a9SDan Williams return rc; 279b64955a9SDan Williams } 280b64955a9SDan Williams 281ed97afb5SBen Widawsky static int cxl_pci_setup_mailbox(struct cxl_mem *cxlm) 28221e9f767SBen Widawsky { 28321e9f767SBen Widawsky const int cap = readl(cxlm->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET); 28421e9f767SBen Widawsky 285b64955a9SDan Williams cxlm->mbox_send = cxl_pci_mbox_send; 28621e9f767SBen Widawsky cxlm->payload_size = 28721e9f767SBen Widawsky 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap); 28821e9f767SBen Widawsky 28921e9f767SBen Widawsky /* 29021e9f767SBen Widawsky * CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register 29121e9f767SBen Widawsky * 29221e9f767SBen Widawsky * If the size is too small, mandatory commands will not work and so 29321e9f767SBen Widawsky * there's no point in going forward. If the size is too large, there's 29421e9f767SBen Widawsky * no harm is soft limiting it. 29521e9f767SBen Widawsky */ 29621e9f767SBen Widawsky cxlm->payload_size = min_t(size_t, cxlm->payload_size, SZ_1M); 29721e9f767SBen Widawsky if (cxlm->payload_size < 256) { 29899e222a5SDan Williams dev_err(cxlm->dev, "Mailbox is too small (%zub)", 29921e9f767SBen Widawsky cxlm->payload_size); 30021e9f767SBen Widawsky return -ENXIO; 30121e9f767SBen Widawsky } 30221e9f767SBen Widawsky 30399e222a5SDan Williams dev_dbg(cxlm->dev, "Mailbox payload sized %zu", 30421e9f767SBen Widawsky cxlm->payload_size); 30521e9f767SBen Widawsky 30621e9f767SBen Widawsky return 0; 30721e9f767SBen Widawsky } 30821e9f767SBen Widawsky 309a261e9a1SDan Williams static int cxl_map_regblock(struct pci_dev *pdev, struct cxl_register_map *map) 3101b0a1a2aSBen Widawsky { 311f8a7e8c2SIra Weiny void __iomem *addr; 3127dc7a64dSBen Widawsky int bar = map->barno; 3137dc7a64dSBen Widawsky struct device *dev = &pdev->dev; 3147dc7a64dSBen Widawsky resource_size_t offset = map->block_offset; 3151b0a1a2aSBen Widawsky 31621e9f767SBen Widawsky /* Basic sanity check that BAR is big enough */ 31721e9f767SBen Widawsky if (pci_resource_len(pdev, bar) < offset) { 3187dc7a64dSBen Widawsky dev_err(dev, "BAR%d: %pr: too small (offset: %pa)\n", bar, 3197dc7a64dSBen Widawsky &pdev->resource[bar], &offset); 320a261e9a1SDan Williams return -ENXIO; 32121e9f767SBen Widawsky } 32221e9f767SBen Widawsky 32330af9729SIra Weiny addr = pci_iomap(pdev, bar, 0); 324f8a7e8c2SIra Weiny if (!addr) { 32521e9f767SBen Widawsky dev_err(dev, "failed to map registers\n"); 326a261e9a1SDan Williams return -ENOMEM; 32721e9f767SBen Widawsky } 32821e9f767SBen Widawsky 3297dc7a64dSBen Widawsky dev_dbg(dev, "Mapped CXL Memory Device resource bar %u @ %pa\n", 3307dc7a64dSBen Widawsky bar, &offset); 3316630d31cSBen Widawsky 332a261e9a1SDan Williams map->base = addr + map->block_offset; 333a261e9a1SDan Williams return 0; 33430af9729SIra Weiny } 33530af9729SIra Weiny 336a261e9a1SDan Williams static void cxl_unmap_regblock(struct pci_dev *pdev, 337a261e9a1SDan Williams struct cxl_register_map *map) 33830af9729SIra Weiny { 339a261e9a1SDan Williams pci_iounmap(pdev, map->base - map->block_offset); 340a261e9a1SDan Williams map->base = NULL; 34121e9f767SBen Widawsky } 34221e9f767SBen Widawsky 343ed97afb5SBen Widawsky static int cxl_pci_dvsec(struct pci_dev *pdev, int dvsec) 34421e9f767SBen Widawsky { 34521e9f767SBen Widawsky int pos; 34621e9f767SBen Widawsky 34721e9f767SBen Widawsky pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DVSEC); 34821e9f767SBen Widawsky if (!pos) 34921e9f767SBen Widawsky return 0; 35021e9f767SBen Widawsky 35121e9f767SBen Widawsky while (pos) { 35221e9f767SBen Widawsky u16 vendor, id; 35321e9f767SBen Widawsky 35421e9f767SBen Widawsky pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER1, &vendor); 35521e9f767SBen Widawsky pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER2, &id); 35621e9f767SBen Widawsky if (vendor == PCI_DVSEC_VENDOR_ID_CXL && dvsec == id) 35721e9f767SBen Widawsky return pos; 35821e9f767SBen Widawsky 35921e9f767SBen Widawsky pos = pci_find_next_ext_capability(pdev, pos, 36021e9f767SBen Widawsky PCI_EXT_CAP_ID_DVSEC); 36121e9f767SBen Widawsky } 36221e9f767SBen Widawsky 36321e9f767SBen Widawsky return 0; 36421e9f767SBen Widawsky } 36521e9f767SBen Widawsky 366a261e9a1SDan Williams static int cxl_probe_regs(struct pci_dev *pdev, struct cxl_register_map *map) 36730af9729SIra Weiny { 36808422378SBen Widawsky struct cxl_component_reg_map *comp_map; 36930af9729SIra Weiny struct cxl_device_reg_map *dev_map; 3707dc7a64dSBen Widawsky struct device *dev = &pdev->dev; 371a261e9a1SDan Williams void __iomem *base = map->base; 37230af9729SIra Weiny 37330af9729SIra Weiny switch (map->reg_type) { 37408422378SBen Widawsky case CXL_REGLOC_RBI_COMPONENT: 37508422378SBen Widawsky comp_map = &map->component_map; 37608422378SBen Widawsky cxl_probe_component_regs(dev, base, comp_map); 37708422378SBen Widawsky if (!comp_map->hdm_decoder.valid) { 37808422378SBen Widawsky dev_err(dev, "HDM decoder registers not found\n"); 37908422378SBen Widawsky return -ENXIO; 38008422378SBen Widawsky } 38108422378SBen Widawsky 38208422378SBen Widawsky dev_dbg(dev, "Set up component registers\n"); 38308422378SBen Widawsky break; 38430af9729SIra Weiny case CXL_REGLOC_RBI_MEMDEV: 38530af9729SIra Weiny dev_map = &map->device_map; 38630af9729SIra Weiny cxl_probe_device_regs(dev, base, dev_map); 38730af9729SIra Weiny if (!dev_map->status.valid || !dev_map->mbox.valid || 38830af9729SIra Weiny !dev_map->memdev.valid) { 38930af9729SIra Weiny dev_err(dev, "registers not found: %s%s%s\n", 39030af9729SIra Weiny !dev_map->status.valid ? "status " : "", 391da582aa5SLi Qiang (Johnny Li) !dev_map->mbox.valid ? "mbox " : "", 392da582aa5SLi Qiang (Johnny Li) !dev_map->memdev.valid ? "memdev " : ""); 39330af9729SIra Weiny return -ENXIO; 39430af9729SIra Weiny } 39530af9729SIra Weiny 39630af9729SIra Weiny dev_dbg(dev, "Probing device registers...\n"); 39730af9729SIra Weiny break; 39830af9729SIra Weiny default: 39930af9729SIra Weiny break; 40030af9729SIra Weiny } 40130af9729SIra Weiny 40230af9729SIra Weiny return 0; 40330af9729SIra Weiny } 40430af9729SIra Weiny 40530af9729SIra Weiny static int cxl_map_regs(struct cxl_mem *cxlm, struct cxl_register_map *map) 40630af9729SIra Weiny { 40799e222a5SDan Williams struct device *dev = cxlm->dev; 40899e222a5SDan Williams struct pci_dev *pdev = to_pci_dev(dev); 40930af9729SIra Weiny 41030af9729SIra Weiny switch (map->reg_type) { 41108422378SBen Widawsky case CXL_REGLOC_RBI_COMPONENT: 41208422378SBen Widawsky cxl_map_component_regs(pdev, &cxlm->regs.component, map); 41308422378SBen Widawsky dev_dbg(dev, "Mapping component registers...\n"); 41408422378SBen Widawsky break; 41530af9729SIra Weiny case CXL_REGLOC_RBI_MEMDEV: 41630af9729SIra Weiny cxl_map_device_regs(pdev, &cxlm->regs.device_regs, map); 41730af9729SIra Weiny dev_dbg(dev, "Probing device registers...\n"); 41830af9729SIra Weiny break; 41930af9729SIra Weiny default: 42030af9729SIra Weiny break; 42130af9729SIra Weiny } 42230af9729SIra Weiny 42330af9729SIra Weiny return 0; 42430af9729SIra Weiny } 42530af9729SIra Weiny 4267dc7a64dSBen Widawsky static void cxl_decode_regblock(u32 reg_lo, u32 reg_hi, 4277dc7a64dSBen Widawsky struct cxl_register_map *map) 42807d62eacSIra Weiny { 4297dc7a64dSBen Widawsky map->block_offset = 4307dc7a64dSBen Widawsky ((u64)reg_hi << 32) | (reg_lo & CXL_REGLOC_ADDR_MASK); 4317dc7a64dSBen Widawsky map->barno = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo); 4327dc7a64dSBen Widawsky map->reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); 43307d62eacSIra Weiny } 43407d62eacSIra Weiny 4351d5a4159SBen Widawsky /** 436*85afc317SBen Widawsky * cxl_find_regblock() - Locate register blocks by type 437*85afc317SBen Widawsky * @pdev: The CXL PCI device to enumerate. 438*85afc317SBen Widawsky * @type: Register Block Indicator id 439*85afc317SBen Widawsky * @map: Enumeration output, clobbered on error 4401d5a4159SBen Widawsky * 441*85afc317SBen Widawsky * Return: 0 if register block enumerated, negative error code otherwise 4421d5a4159SBen Widawsky * 443*85afc317SBen Widawsky * A CXL DVSEC may point to one or more register blocks, search for them 444*85afc317SBen Widawsky * by @type. 4451d5a4159SBen Widawsky */ 446*85afc317SBen Widawsky static int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type, 447*85afc317SBen Widawsky struct cxl_register_map *map) 4481d5a4159SBen Widawsky { 44999e222a5SDan Williams u32 regloc_size, regblocks; 450*85afc317SBen Widawsky int regloc, i; 4511d5a4159SBen Widawsky 452ed97afb5SBen Widawsky regloc = cxl_pci_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_DVSEC_ID); 453*85afc317SBen Widawsky if (!regloc) 4541d5a4159SBen Widawsky return -ENXIO; 4551d5a4159SBen Widawsky 4561d5a4159SBen Widawsky pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size); 4571d5a4159SBen Widawsky regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size); 4581d5a4159SBen Widawsky 4591d5a4159SBen Widawsky regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET; 4601d5a4159SBen Widawsky regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8; 4611d5a4159SBen Widawsky 462*85afc317SBen Widawsky for (i = 0; i < regblocks; i++, regloc += 8) { 4631d5a4159SBen Widawsky u32 reg_lo, reg_hi; 4641d5a4159SBen Widawsky 4651d5a4159SBen Widawsky pci_read_config_dword(pdev, regloc, ®_lo); 4661d5a4159SBen Widawsky pci_read_config_dword(pdev, regloc + 4, ®_hi); 4671d5a4159SBen Widawsky 4687dc7a64dSBen Widawsky cxl_decode_regblock(reg_lo, reg_hi, map); 46907d62eacSIra Weiny 470*85afc317SBen Widawsky if (map->reg_type == type) 471*85afc317SBen Widawsky return 0; 472*85afc317SBen Widawsky } 4731e39db57SBen Widawsky 474*85afc317SBen Widawsky return -ENODEV; 475*85afc317SBen Widawsky } 4761d5a4159SBen Widawsky 477*85afc317SBen Widawsky static int cxl_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type, 478*85afc317SBen Widawsky struct cxl_register_map *map) 479*85afc317SBen Widawsky { 480*85afc317SBen Widawsky int rc; 481*85afc317SBen Widawsky 482*85afc317SBen Widawsky rc = cxl_find_regblock(pdev, type, map); 483*85afc317SBen Widawsky if (rc) 484*85afc317SBen Widawsky return rc; 485*85afc317SBen Widawsky 486*85afc317SBen Widawsky rc = cxl_map_regblock(pdev, map); 487*85afc317SBen Widawsky if (rc) 488*85afc317SBen Widawsky return rc; 489*85afc317SBen Widawsky 490*85afc317SBen Widawsky rc = cxl_probe_regs(pdev, map); 491a261e9a1SDan Williams cxl_unmap_regblock(pdev, map); 4925b68705dSBen Widawsky 493*85afc317SBen Widawsky return rc; 4941d5a4159SBen Widawsky } 4951d5a4159SBen Widawsky 496ed97afb5SBen Widawsky static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 49721e9f767SBen Widawsky { 498*85afc317SBen Widawsky struct cxl_register_map map; 49921083f51SDan Williams struct cxl_memdev *cxlmd; 5001b0a1a2aSBen Widawsky struct cxl_mem *cxlm; 5011d5a4159SBen Widawsky int rc; 50221e9f767SBen Widawsky 5035a2328f4SDan Williams /* 5045a2328f4SDan Williams * Double check the anonymous union trickery in struct cxl_regs 5055a2328f4SDan Williams * FIXME switch to struct_group() 5065a2328f4SDan Williams */ 5075a2328f4SDan Williams BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) != 5085a2328f4SDan Williams offsetof(struct cxl_regs, device_regs.memdev)); 5095a2328f4SDan Williams 51021e9f767SBen Widawsky rc = pcim_enable_device(pdev); 51121e9f767SBen Widawsky if (rc) 51221e9f767SBen Widawsky return rc; 51321e9f767SBen Widawsky 51499e222a5SDan Williams cxlm = cxl_mem_create(&pdev->dev); 5151b0a1a2aSBen Widawsky if (IS_ERR(cxlm)) 5161b0a1a2aSBen Widawsky return PTR_ERR(cxlm); 5171b0a1a2aSBen Widawsky 518*85afc317SBen Widawsky rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map); 519*85afc317SBen Widawsky if (rc) 520*85afc317SBen Widawsky return rc; 521*85afc317SBen Widawsky 522*85afc317SBen Widawsky rc = cxl_map_regs(cxlm, &map); 52321e9f767SBen Widawsky if (rc) 52421e9f767SBen Widawsky return rc; 52521e9f767SBen Widawsky 526ed97afb5SBen Widawsky rc = cxl_pci_setup_mailbox(cxlm); 52721e9f767SBen Widawsky if (rc) 52821e9f767SBen Widawsky return rc; 52921e9f767SBen Widawsky 53021e9f767SBen Widawsky rc = cxl_mem_enumerate_cmds(cxlm); 53121e9f767SBen Widawsky if (rc) 53221e9f767SBen Widawsky return rc; 53321e9f767SBen Widawsky 53421e9f767SBen Widawsky rc = cxl_mem_identify(cxlm); 53521e9f767SBen Widawsky if (rc) 53621e9f767SBen Widawsky return rc; 53721e9f767SBen Widawsky 538f847502aSIra Weiny rc = cxl_mem_create_range_info(cxlm); 539f847502aSIra Weiny if (rc) 540f847502aSIra Weiny return rc; 541f847502aSIra Weiny 5424faf31b4SDan Williams cxlmd = devm_cxl_add_memdev(cxlm); 54321083f51SDan Williams if (IS_ERR(cxlmd)) 54421083f51SDan Williams return PTR_ERR(cxlmd); 54521083f51SDan Williams 54621083f51SDan Williams if (range_len(&cxlm->pmem_range) && IS_ENABLED(CONFIG_CXL_PMEM)) 54721083f51SDan Williams rc = devm_cxl_add_nvdimm(&pdev->dev, cxlmd); 54821083f51SDan Williams 54921083f51SDan Williams return rc; 55021e9f767SBen Widawsky } 55121e9f767SBen Widawsky 55221e9f767SBen Widawsky static const struct pci_device_id cxl_mem_pci_tbl[] = { 55321e9f767SBen Widawsky /* PCI class code for CXL.mem Type-3 Devices */ 55421e9f767SBen Widawsky { PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)}, 55521e9f767SBen Widawsky { /* terminate list */ }, 55621e9f767SBen Widawsky }; 55721e9f767SBen Widawsky MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl); 55821e9f767SBen Widawsky 559ed97afb5SBen Widawsky static struct pci_driver cxl_pci_driver = { 56021e9f767SBen Widawsky .name = KBUILD_MODNAME, 56121e9f767SBen Widawsky .id_table = cxl_mem_pci_tbl, 562ed97afb5SBen Widawsky .probe = cxl_pci_probe, 56321e9f767SBen Widawsky .driver = { 56421e9f767SBen Widawsky .probe_type = PROBE_PREFER_ASYNCHRONOUS, 56521e9f767SBen Widawsky }, 56621e9f767SBen Widawsky }; 56721e9f767SBen Widawsky 56821e9f767SBen Widawsky MODULE_LICENSE("GPL v2"); 569ed97afb5SBen Widawsky module_pci_driver(cxl_pci_driver); 57021e9f767SBen Widawsky MODULE_IMPORT_NS(CXL); 571