121e9f767SBen Widawsky // SPDX-License-Identifier: GPL-2.0-only 221e9f767SBen Widawsky /* Copyright(c) 2020 Intel Corporation. All rights reserved. */ 321e9f767SBen Widawsky #include <uapi/linux/cxl_mem.h> 421e9f767SBen Widawsky #include <linux/security.h> 521e9f767SBen Widawsky #include <linux/debugfs.h> 621e9f767SBen Widawsky #include <linux/module.h> 721e9f767SBen Widawsky #include <linux/sizes.h> 821e9f767SBen Widawsky #include <linux/mutex.h> 921e9f767SBen Widawsky #include <linux/cdev.h> 1021e9f767SBen Widawsky #include <linux/idr.h> 1121e9f767SBen Widawsky #include <linux/pci.h> 1221e9f767SBen Widawsky #include <linux/io.h> 1321e9f767SBen Widawsky #include <linux/io-64-nonatomic-lo-hi.h> 1421e9f767SBen Widawsky #include "pci.h" 1521e9f767SBen Widawsky #include "cxl.h" 1621e9f767SBen Widawsky #include "mem.h" 1721e9f767SBen Widawsky 1821e9f767SBen Widawsky /** 1921e9f767SBen Widawsky * DOC: cxl pci 2021e9f767SBen Widawsky * 2121e9f767SBen Widawsky * This implements the PCI exclusive functionality for a CXL device as it is 2221e9f767SBen Widawsky * defined by the Compute Express Link specification. CXL devices may surface 2321e9f767SBen Widawsky * certain functionality even if it isn't CXL enabled. 2421e9f767SBen Widawsky * 2521e9f767SBen Widawsky * The driver has several responsibilities, mainly: 2621e9f767SBen Widawsky * - Create the memX device and register on the CXL bus. 2721e9f767SBen Widawsky * - Enumerate device's register interface and map them. 2821e9f767SBen Widawsky * - Probe the device attributes to establish sysfs interface. 2921e9f767SBen Widawsky * - Provide an IOCTL interface to userspace to communicate with the device for 3021e9f767SBen Widawsky * things like firmware update. 3121e9f767SBen Widawsky */ 3221e9f767SBen Widawsky 3321e9f767SBen Widawsky #define cxl_doorbell_busy(cxlm) \ 3421e9f767SBen Widawsky (readl((cxlm)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \ 3521e9f767SBen Widawsky CXLDEV_MBOX_CTRL_DOORBELL) 3621e9f767SBen Widawsky 3721e9f767SBen Widawsky /* CXL 2.0 - 8.2.8.4 */ 3821e9f767SBen Widawsky #define CXL_MAILBOX_TIMEOUT_MS (2 * HZ) 3921e9f767SBen Widawsky 4021e9f767SBen Widawsky enum opcode { 4121e9f767SBen Widawsky CXL_MBOX_OP_INVALID = 0x0000, 4221e9f767SBen Widawsky CXL_MBOX_OP_RAW = CXL_MBOX_OP_INVALID, 4321e9f767SBen Widawsky CXL_MBOX_OP_GET_FW_INFO = 0x0200, 4421e9f767SBen Widawsky CXL_MBOX_OP_ACTIVATE_FW = 0x0202, 4521e9f767SBen Widawsky CXL_MBOX_OP_GET_SUPPORTED_LOGS = 0x0400, 4621e9f767SBen Widawsky CXL_MBOX_OP_GET_LOG = 0x0401, 4721e9f767SBen Widawsky CXL_MBOX_OP_IDENTIFY = 0x4000, 4821e9f767SBen Widawsky CXL_MBOX_OP_GET_PARTITION_INFO = 0x4100, 4921e9f767SBen Widawsky CXL_MBOX_OP_SET_PARTITION_INFO = 0x4101, 5021e9f767SBen Widawsky CXL_MBOX_OP_GET_LSA = 0x4102, 5121e9f767SBen Widawsky CXL_MBOX_OP_SET_LSA = 0x4103, 5221e9f767SBen Widawsky CXL_MBOX_OP_GET_HEALTH_INFO = 0x4200, 5321e9f767SBen Widawsky CXL_MBOX_OP_SET_SHUTDOWN_STATE = 0x4204, 5421e9f767SBen Widawsky CXL_MBOX_OP_SCAN_MEDIA = 0x4304, 5521e9f767SBen Widawsky CXL_MBOX_OP_GET_SCAN_MEDIA = 0x4305, 5621e9f767SBen Widawsky CXL_MBOX_OP_MAX = 0x10000 5721e9f767SBen Widawsky }; 5821e9f767SBen Widawsky 5921e9f767SBen Widawsky /** 6021e9f767SBen Widawsky * struct mbox_cmd - A command to be submitted to hardware. 6121e9f767SBen Widawsky * @opcode: (input) The command set and command submitted to hardware. 6221e9f767SBen Widawsky * @payload_in: (input) Pointer to the input payload. 6321e9f767SBen Widawsky * @payload_out: (output) Pointer to the output payload. Must be allocated by 6421e9f767SBen Widawsky * the caller. 6521e9f767SBen Widawsky * @size_in: (input) Number of bytes to load from @payload_in. 6621e9f767SBen Widawsky * @size_out: (input) Max number of bytes loaded into @payload_out. 6721e9f767SBen Widawsky * (output) Number of bytes generated by the device. For fixed size 6821e9f767SBen Widawsky * outputs commands this is always expected to be deterministic. For 6921e9f767SBen Widawsky * variable sized output commands, it tells the exact number of bytes 7021e9f767SBen Widawsky * written. 7121e9f767SBen Widawsky * @return_code: (output) Error code returned from hardware. 7221e9f767SBen Widawsky * 7321e9f767SBen Widawsky * This is the primary mechanism used to send commands to the hardware. 7421e9f767SBen Widawsky * All the fields except @payload_* correspond exactly to the fields described in 7521e9f767SBen Widawsky * Command Register section of the CXL 2.0 8.2.8.4.5. @payload_in and 7621e9f767SBen Widawsky * @payload_out are written to, and read from the Command Payload Registers 7721e9f767SBen Widawsky * defined in CXL 2.0 8.2.8.4.8. 7821e9f767SBen Widawsky */ 7921e9f767SBen Widawsky struct mbox_cmd { 8021e9f767SBen Widawsky u16 opcode; 8121e9f767SBen Widawsky void *payload_in; 8221e9f767SBen Widawsky void *payload_out; 8321e9f767SBen Widawsky size_t size_in; 8421e9f767SBen Widawsky size_t size_out; 8521e9f767SBen Widawsky u16 return_code; 8621e9f767SBen Widawsky #define CXL_MBOX_SUCCESS 0 8721e9f767SBen Widawsky }; 8821e9f767SBen Widawsky 8921e9f767SBen Widawsky static int cxl_mem_major; 9021e9f767SBen Widawsky static DEFINE_IDA(cxl_memdev_ida); 9121e9f767SBen Widawsky static DECLARE_RWSEM(cxl_memdev_rwsem); 9221e9f767SBen Widawsky static struct dentry *cxl_debugfs; 9321e9f767SBen Widawsky static bool cxl_raw_allow_all; 9421e9f767SBen Widawsky 9521e9f767SBen Widawsky enum { 9621e9f767SBen Widawsky CEL_UUID, 9721e9f767SBen Widawsky VENDOR_DEBUG_UUID, 9821e9f767SBen Widawsky }; 9921e9f767SBen Widawsky 10021e9f767SBen Widawsky /* See CXL 2.0 Table 170. Get Log Input Payload */ 10121e9f767SBen Widawsky static const uuid_t log_uuid[] = { 10221e9f767SBen Widawsky [CEL_UUID] = UUID_INIT(0xda9c0b5, 0xbf41, 0x4b78, 0x8f, 0x79, 0x96, 10321e9f767SBen Widawsky 0xb1, 0x62, 0x3b, 0x3f, 0x17), 10421e9f767SBen Widawsky [VENDOR_DEBUG_UUID] = UUID_INIT(0xe1819d9, 0x11a9, 0x400c, 0x81, 0x1f, 10521e9f767SBen Widawsky 0xd6, 0x07, 0x19, 0x40, 0x3d, 0x86), 10621e9f767SBen Widawsky }; 10721e9f767SBen Widawsky 10821e9f767SBen Widawsky /** 10921e9f767SBen Widawsky * struct cxl_mem_command - Driver representation of a memory device command 11021e9f767SBen Widawsky * @info: Command information as it exists for the UAPI 11121e9f767SBen Widawsky * @opcode: The actual bits used for the mailbox protocol 11221e9f767SBen Widawsky * @flags: Set of flags effecting driver behavior. 11321e9f767SBen Widawsky * 11421e9f767SBen Widawsky * * %CXL_CMD_FLAG_FORCE_ENABLE: In cases of error, commands with this flag 11521e9f767SBen Widawsky * will be enabled by the driver regardless of what hardware may have 11621e9f767SBen Widawsky * advertised. 11721e9f767SBen Widawsky * 11821e9f767SBen Widawsky * The cxl_mem_command is the driver's internal representation of commands that 11921e9f767SBen Widawsky * are supported by the driver. Some of these commands may not be supported by 12021e9f767SBen Widawsky * the hardware. The driver will use @info to validate the fields passed in by 12121e9f767SBen Widawsky * the user then submit the @opcode to the hardware. 12221e9f767SBen Widawsky * 12321e9f767SBen Widawsky * See struct cxl_command_info. 12421e9f767SBen Widawsky */ 12521e9f767SBen Widawsky struct cxl_mem_command { 12621e9f767SBen Widawsky struct cxl_command_info info; 12721e9f767SBen Widawsky enum opcode opcode; 12821e9f767SBen Widawsky u32 flags; 12921e9f767SBen Widawsky #define CXL_CMD_FLAG_NONE 0 13021e9f767SBen Widawsky #define CXL_CMD_FLAG_FORCE_ENABLE BIT(0) 13121e9f767SBen Widawsky }; 13221e9f767SBen Widawsky 13321e9f767SBen Widawsky #define CXL_CMD(_id, sin, sout, _flags) \ 13421e9f767SBen Widawsky [CXL_MEM_COMMAND_ID_##_id] = { \ 13521e9f767SBen Widawsky .info = { \ 13621e9f767SBen Widawsky .id = CXL_MEM_COMMAND_ID_##_id, \ 13721e9f767SBen Widawsky .size_in = sin, \ 13821e9f767SBen Widawsky .size_out = sout, \ 13921e9f767SBen Widawsky }, \ 14021e9f767SBen Widawsky .opcode = CXL_MBOX_OP_##_id, \ 14121e9f767SBen Widawsky .flags = _flags, \ 14221e9f767SBen Widawsky } 14321e9f767SBen Widawsky 14421e9f767SBen Widawsky /* 14521e9f767SBen Widawsky * This table defines the supported mailbox commands for the driver. This table 14621e9f767SBen Widawsky * is made up of a UAPI structure. Non-negative values as parameters in the 14721e9f767SBen Widawsky * table will be validated against the user's input. For example, if size_in is 14821e9f767SBen Widawsky * 0, and the user passed in 1, it is an error. 14921e9f767SBen Widawsky */ 15021e9f767SBen Widawsky static struct cxl_mem_command mem_commands[CXL_MEM_COMMAND_ID_MAX] = { 15121e9f767SBen Widawsky CXL_CMD(IDENTIFY, 0, 0x43, CXL_CMD_FLAG_FORCE_ENABLE), 15221e9f767SBen Widawsky #ifdef CONFIG_CXL_MEM_RAW_COMMANDS 15321e9f767SBen Widawsky CXL_CMD(RAW, ~0, ~0, 0), 15421e9f767SBen Widawsky #endif 15521e9f767SBen Widawsky CXL_CMD(GET_SUPPORTED_LOGS, 0, ~0, CXL_CMD_FLAG_FORCE_ENABLE), 15621e9f767SBen Widawsky CXL_CMD(GET_FW_INFO, 0, 0x50, 0), 15721e9f767SBen Widawsky CXL_CMD(GET_PARTITION_INFO, 0, 0x20, 0), 15821e9f767SBen Widawsky CXL_CMD(GET_LSA, 0x8, ~0, 0), 15921e9f767SBen Widawsky CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0), 16021e9f767SBen Widawsky CXL_CMD(GET_LOG, 0x18, ~0, CXL_CMD_FLAG_FORCE_ENABLE), 16121e9f767SBen Widawsky }; 16221e9f767SBen Widawsky 16321e9f767SBen Widawsky /* 16421e9f767SBen Widawsky * Commands that RAW doesn't permit. The rationale for each: 16521e9f767SBen Widawsky * 16621e9f767SBen Widawsky * CXL_MBOX_OP_ACTIVATE_FW: Firmware activation requires adjustment / 16721e9f767SBen Widawsky * coordination of transaction timeout values at the root bridge level. 16821e9f767SBen Widawsky * 16921e9f767SBen Widawsky * CXL_MBOX_OP_SET_PARTITION_INFO: The device memory map may change live 17021e9f767SBen Widawsky * and needs to be coordinated with HDM updates. 17121e9f767SBen Widawsky * 17221e9f767SBen Widawsky * CXL_MBOX_OP_SET_LSA: The label storage area may be cached by the 17321e9f767SBen Widawsky * driver and any writes from userspace invalidates those contents. 17421e9f767SBen Widawsky * 17521e9f767SBen Widawsky * CXL_MBOX_OP_SET_SHUTDOWN_STATE: Set shutdown state assumes no writes 17621e9f767SBen Widawsky * to the device after it is marked clean, userspace can not make that 17721e9f767SBen Widawsky * assertion. 17821e9f767SBen Widawsky * 17921e9f767SBen Widawsky * CXL_MBOX_OP_[GET_]SCAN_MEDIA: The kernel provides a native error list that 18021e9f767SBen Widawsky * is kept up to date with patrol notifications and error management. 18121e9f767SBen Widawsky */ 18221e9f767SBen Widawsky static u16 cxl_disabled_raw_commands[] = { 18321e9f767SBen Widawsky CXL_MBOX_OP_ACTIVATE_FW, 18421e9f767SBen Widawsky CXL_MBOX_OP_SET_PARTITION_INFO, 18521e9f767SBen Widawsky CXL_MBOX_OP_SET_LSA, 18621e9f767SBen Widawsky CXL_MBOX_OP_SET_SHUTDOWN_STATE, 18721e9f767SBen Widawsky CXL_MBOX_OP_SCAN_MEDIA, 18821e9f767SBen Widawsky CXL_MBOX_OP_GET_SCAN_MEDIA, 18921e9f767SBen Widawsky }; 19021e9f767SBen Widawsky 19121e9f767SBen Widawsky /* 19221e9f767SBen Widawsky * Command sets that RAW doesn't permit. All opcodes in this set are 19321e9f767SBen Widawsky * disabled because they pass plain text security payloads over the 19421e9f767SBen Widawsky * user/kernel boundary. This functionality is intended to be wrapped 19521e9f767SBen Widawsky * behind the keys ABI which allows for encrypted payloads in the UAPI 19621e9f767SBen Widawsky */ 19721e9f767SBen Widawsky static u8 security_command_sets[] = { 19821e9f767SBen Widawsky 0x44, /* Sanitize */ 19921e9f767SBen Widawsky 0x45, /* Persistent Memory Data-at-rest Security */ 20021e9f767SBen Widawsky 0x46, /* Security Passthrough */ 20121e9f767SBen Widawsky }; 20221e9f767SBen Widawsky 20321e9f767SBen Widawsky #define cxl_for_each_cmd(cmd) \ 20421e9f767SBen Widawsky for ((cmd) = &mem_commands[0]; \ 20521e9f767SBen Widawsky ((cmd) - mem_commands) < ARRAY_SIZE(mem_commands); (cmd)++) 20621e9f767SBen Widawsky 20721e9f767SBen Widawsky #define cxl_cmd_count ARRAY_SIZE(mem_commands) 20821e9f767SBen Widawsky 20921e9f767SBen Widawsky static int cxl_mem_wait_for_doorbell(struct cxl_mem *cxlm) 21021e9f767SBen Widawsky { 21121e9f767SBen Widawsky const unsigned long start = jiffies; 21221e9f767SBen Widawsky unsigned long end = start; 21321e9f767SBen Widawsky 21421e9f767SBen Widawsky while (cxl_doorbell_busy(cxlm)) { 21521e9f767SBen Widawsky end = jiffies; 21621e9f767SBen Widawsky 21721e9f767SBen Widawsky if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) { 21821e9f767SBen Widawsky /* Check again in case preempted before timeout test */ 21921e9f767SBen Widawsky if (!cxl_doorbell_busy(cxlm)) 22021e9f767SBen Widawsky break; 22121e9f767SBen Widawsky return -ETIMEDOUT; 22221e9f767SBen Widawsky } 22321e9f767SBen Widawsky cpu_relax(); 22421e9f767SBen Widawsky } 22521e9f767SBen Widawsky 22621e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, "Doorbell wait took %dms", 22721e9f767SBen Widawsky jiffies_to_msecs(end) - jiffies_to_msecs(start)); 22821e9f767SBen Widawsky return 0; 22921e9f767SBen Widawsky } 23021e9f767SBen Widawsky 23121e9f767SBen Widawsky static bool cxl_is_security_command(u16 opcode) 23221e9f767SBen Widawsky { 23321e9f767SBen Widawsky int i; 23421e9f767SBen Widawsky 23521e9f767SBen Widawsky for (i = 0; i < ARRAY_SIZE(security_command_sets); i++) 23621e9f767SBen Widawsky if (security_command_sets[i] == (opcode >> 8)) 23721e9f767SBen Widawsky return true; 23821e9f767SBen Widawsky return false; 23921e9f767SBen Widawsky } 24021e9f767SBen Widawsky 24121e9f767SBen Widawsky static void cxl_mem_mbox_timeout(struct cxl_mem *cxlm, 24221e9f767SBen Widawsky struct mbox_cmd *mbox_cmd) 24321e9f767SBen Widawsky { 24421e9f767SBen Widawsky struct device *dev = &cxlm->pdev->dev; 24521e9f767SBen Widawsky 24621e9f767SBen Widawsky dev_dbg(dev, "Mailbox command (opcode: %#x size: %zub) timed out\n", 24721e9f767SBen Widawsky mbox_cmd->opcode, mbox_cmd->size_in); 24821e9f767SBen Widawsky } 24921e9f767SBen Widawsky 25021e9f767SBen Widawsky /** 25121e9f767SBen Widawsky * __cxl_mem_mbox_send_cmd() - Execute a mailbox command 25221e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 25321e9f767SBen Widawsky * @mbox_cmd: Command to send to the memory device. 25421e9f767SBen Widawsky * 25521e9f767SBen Widawsky * Context: Any context. Expects mbox_mutex to be held. 25621e9f767SBen Widawsky * Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success. 25721e9f767SBen Widawsky * Caller should check the return code in @mbox_cmd to make sure it 25821e9f767SBen Widawsky * succeeded. 25921e9f767SBen Widawsky * 26021e9f767SBen Widawsky * This is a generic form of the CXL mailbox send command thus only using the 26121e9f767SBen Widawsky * registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory 26221e9f767SBen Widawsky * devices, and perhaps other types of CXL devices may have further information 26321e9f767SBen Widawsky * available upon error conditions. Driver facilities wishing to send mailbox 26421e9f767SBen Widawsky * commands should use the wrapper command. 26521e9f767SBen Widawsky * 26621e9f767SBen Widawsky * The CXL spec allows for up to two mailboxes. The intention is for the primary 26721e9f767SBen Widawsky * mailbox to be OS controlled and the secondary mailbox to be used by system 26821e9f767SBen Widawsky * firmware. This allows the OS and firmware to communicate with the device and 26921e9f767SBen Widawsky * not need to coordinate with each other. The driver only uses the primary 27021e9f767SBen Widawsky * mailbox. 27121e9f767SBen Widawsky */ 27221e9f767SBen Widawsky static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, 27321e9f767SBen Widawsky struct mbox_cmd *mbox_cmd) 27421e9f767SBen Widawsky { 27521e9f767SBen Widawsky void __iomem *payload = cxlm->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET; 27621e9f767SBen Widawsky u64 cmd_reg, status_reg; 27721e9f767SBen Widawsky size_t out_len; 27821e9f767SBen Widawsky int rc; 27921e9f767SBen Widawsky 28021e9f767SBen Widawsky lockdep_assert_held(&cxlm->mbox_mutex); 28121e9f767SBen Widawsky 28221e9f767SBen Widawsky /* 28321e9f767SBen Widawsky * Here are the steps from 8.2.8.4 of the CXL 2.0 spec. 28421e9f767SBen Widawsky * 1. Caller reads MB Control Register to verify doorbell is clear 28521e9f767SBen Widawsky * 2. Caller writes Command Register 28621e9f767SBen Widawsky * 3. Caller writes Command Payload Registers if input payload is non-empty 28721e9f767SBen Widawsky * 4. Caller writes MB Control Register to set doorbell 28821e9f767SBen Widawsky * 5. Caller either polls for doorbell to be clear or waits for interrupt if configured 28921e9f767SBen Widawsky * 6. Caller reads MB Status Register to fetch Return code 29021e9f767SBen Widawsky * 7. If command successful, Caller reads Command Register to get Payload Length 29121e9f767SBen Widawsky * 8. If output payload is non-empty, host reads Command Payload Registers 29221e9f767SBen Widawsky * 29321e9f767SBen Widawsky * Hardware is free to do whatever it wants before the doorbell is rung, 29421e9f767SBen Widawsky * and isn't allowed to change anything after it clears the doorbell. As 29521e9f767SBen Widawsky * such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can 29621e9f767SBen Widawsky * also happen in any order (though some orders might not make sense). 29721e9f767SBen Widawsky */ 29821e9f767SBen Widawsky 29921e9f767SBen Widawsky /* #1 */ 30021e9f767SBen Widawsky if (cxl_doorbell_busy(cxlm)) { 30121e9f767SBen Widawsky dev_err_ratelimited(&cxlm->pdev->dev, 30221e9f767SBen Widawsky "Mailbox re-busy after acquiring\n"); 30321e9f767SBen Widawsky return -EBUSY; 30421e9f767SBen Widawsky } 30521e9f767SBen Widawsky 30621e9f767SBen Widawsky cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK, 30721e9f767SBen Widawsky mbox_cmd->opcode); 30821e9f767SBen Widawsky if (mbox_cmd->size_in) { 30921e9f767SBen Widawsky if (WARN_ON(!mbox_cmd->payload_in)) 31021e9f767SBen Widawsky return -EINVAL; 31121e9f767SBen Widawsky 31221e9f767SBen Widawsky cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, 31321e9f767SBen Widawsky mbox_cmd->size_in); 31421e9f767SBen Widawsky memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in); 31521e9f767SBen Widawsky } 31621e9f767SBen Widawsky 31721e9f767SBen Widawsky /* #2, #3 */ 31821e9f767SBen Widawsky writeq(cmd_reg, cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 31921e9f767SBen Widawsky 32021e9f767SBen Widawsky /* #4 */ 32121e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, "Sending command\n"); 32221e9f767SBen Widawsky writel(CXLDEV_MBOX_CTRL_DOORBELL, 32321e9f767SBen Widawsky cxlm->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET); 32421e9f767SBen Widawsky 32521e9f767SBen Widawsky /* #5 */ 32621e9f767SBen Widawsky rc = cxl_mem_wait_for_doorbell(cxlm); 32721e9f767SBen Widawsky if (rc == -ETIMEDOUT) { 32821e9f767SBen Widawsky cxl_mem_mbox_timeout(cxlm, mbox_cmd); 32921e9f767SBen Widawsky return rc; 33021e9f767SBen Widawsky } 33121e9f767SBen Widawsky 33221e9f767SBen Widawsky /* #6 */ 33321e9f767SBen Widawsky status_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET); 33421e9f767SBen Widawsky mbox_cmd->return_code = 33521e9f767SBen Widawsky FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg); 33621e9f767SBen Widawsky 33721e9f767SBen Widawsky if (mbox_cmd->return_code != 0) { 33821e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, "Mailbox operation had an error\n"); 33921e9f767SBen Widawsky return 0; 34021e9f767SBen Widawsky } 34121e9f767SBen Widawsky 34221e9f767SBen Widawsky /* #7 */ 34321e9f767SBen Widawsky cmd_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 34421e9f767SBen Widawsky out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg); 34521e9f767SBen Widawsky 34621e9f767SBen Widawsky /* #8 */ 34721e9f767SBen Widawsky if (out_len && mbox_cmd->payload_out) { 34821e9f767SBen Widawsky /* 34921e9f767SBen Widawsky * Sanitize the copy. If hardware misbehaves, out_len per the 35021e9f767SBen Widawsky * spec can actually be greater than the max allowed size (21 35121e9f767SBen Widawsky * bits available but spec defined 1M max). The caller also may 35221e9f767SBen Widawsky * have requested less data than the hardware supplied even 35321e9f767SBen Widawsky * within spec. 35421e9f767SBen Widawsky */ 35521e9f767SBen Widawsky size_t n = min3(mbox_cmd->size_out, cxlm->payload_size, out_len); 35621e9f767SBen Widawsky 35721e9f767SBen Widawsky memcpy_fromio(mbox_cmd->payload_out, payload, n); 35821e9f767SBen Widawsky mbox_cmd->size_out = n; 35921e9f767SBen Widawsky } else { 36021e9f767SBen Widawsky mbox_cmd->size_out = 0; 36121e9f767SBen Widawsky } 36221e9f767SBen Widawsky 36321e9f767SBen Widawsky return 0; 36421e9f767SBen Widawsky } 36521e9f767SBen Widawsky 36621e9f767SBen Widawsky /** 36721e9f767SBen Widawsky * cxl_mem_mbox_get() - Acquire exclusive access to the mailbox. 36821e9f767SBen Widawsky * @cxlm: The memory device to gain access to. 36921e9f767SBen Widawsky * 37021e9f767SBen Widawsky * Context: Any context. Takes the mbox_mutex. 37121e9f767SBen Widawsky * Return: 0 if exclusive access was acquired. 37221e9f767SBen Widawsky */ 37321e9f767SBen Widawsky static int cxl_mem_mbox_get(struct cxl_mem *cxlm) 37421e9f767SBen Widawsky { 37521e9f767SBen Widawsky struct device *dev = &cxlm->pdev->dev; 37621e9f767SBen Widawsky u64 md_status; 37721e9f767SBen Widawsky int rc; 37821e9f767SBen Widawsky 37921e9f767SBen Widawsky mutex_lock_io(&cxlm->mbox_mutex); 38021e9f767SBen Widawsky 38121e9f767SBen Widawsky /* 38221e9f767SBen Widawsky * XXX: There is some amount of ambiguity in the 2.0 version of the spec 38321e9f767SBen Widawsky * around the mailbox interface ready (8.2.8.5.1.1). The purpose of the 38421e9f767SBen Widawsky * bit is to allow firmware running on the device to notify the driver 38521e9f767SBen Widawsky * that it's ready to receive commands. It is unclear if the bit needs 38621e9f767SBen Widawsky * to be read for each transaction mailbox, ie. the firmware can switch 38721e9f767SBen Widawsky * it on and off as needed. Second, there is no defined timeout for 38821e9f767SBen Widawsky * mailbox ready, like there is for the doorbell interface. 38921e9f767SBen Widawsky * 39021e9f767SBen Widawsky * Assumptions: 39121e9f767SBen Widawsky * 1. The firmware might toggle the Mailbox Interface Ready bit, check 39221e9f767SBen Widawsky * it for every command. 39321e9f767SBen Widawsky * 39421e9f767SBen Widawsky * 2. If the doorbell is clear, the firmware should have first set the 39521e9f767SBen Widawsky * Mailbox Interface Ready bit. Therefore, waiting for the doorbell 39621e9f767SBen Widawsky * to be ready is sufficient. 39721e9f767SBen Widawsky */ 39821e9f767SBen Widawsky rc = cxl_mem_wait_for_doorbell(cxlm); 39921e9f767SBen Widawsky if (rc) { 40021e9f767SBen Widawsky dev_warn(dev, "Mailbox interface not ready\n"); 40121e9f767SBen Widawsky goto out; 40221e9f767SBen Widawsky } 40321e9f767SBen Widawsky 40421e9f767SBen Widawsky md_status = readq(cxlm->regs.memdev + CXLMDEV_STATUS_OFFSET); 40521e9f767SBen Widawsky if (!(md_status & CXLMDEV_MBOX_IF_READY && CXLMDEV_READY(md_status))) { 40621e9f767SBen Widawsky dev_err(dev, "mbox: reported doorbell ready, but not mbox ready\n"); 40721e9f767SBen Widawsky rc = -EBUSY; 40821e9f767SBen Widawsky goto out; 40921e9f767SBen Widawsky } 41021e9f767SBen Widawsky 41121e9f767SBen Widawsky /* 41221e9f767SBen Widawsky * Hardware shouldn't allow a ready status but also have failure bits 41321e9f767SBen Widawsky * set. Spit out an error, this should be a bug report 41421e9f767SBen Widawsky */ 41521e9f767SBen Widawsky rc = -EFAULT; 41621e9f767SBen Widawsky if (md_status & CXLMDEV_DEV_FATAL) { 41721e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but fatal\n"); 41821e9f767SBen Widawsky goto out; 41921e9f767SBen Widawsky } 42021e9f767SBen Widawsky if (md_status & CXLMDEV_FW_HALT) { 42121e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but halted\n"); 42221e9f767SBen Widawsky goto out; 42321e9f767SBen Widawsky } 42421e9f767SBen Widawsky if (CXLMDEV_RESET_NEEDED(md_status)) { 42521e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but reset needed\n"); 42621e9f767SBen Widawsky goto out; 42721e9f767SBen Widawsky } 42821e9f767SBen Widawsky 42921e9f767SBen Widawsky /* with lock held */ 43021e9f767SBen Widawsky return 0; 43121e9f767SBen Widawsky 43221e9f767SBen Widawsky out: 43321e9f767SBen Widawsky mutex_unlock(&cxlm->mbox_mutex); 43421e9f767SBen Widawsky return rc; 43521e9f767SBen Widawsky } 43621e9f767SBen Widawsky 43721e9f767SBen Widawsky /** 43821e9f767SBen Widawsky * cxl_mem_mbox_put() - Release exclusive access to the mailbox. 43921e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 44021e9f767SBen Widawsky * 44121e9f767SBen Widawsky * Context: Any context. Expects mbox_mutex to be held. 44221e9f767SBen Widawsky */ 44321e9f767SBen Widawsky static void cxl_mem_mbox_put(struct cxl_mem *cxlm) 44421e9f767SBen Widawsky { 44521e9f767SBen Widawsky mutex_unlock(&cxlm->mbox_mutex); 44621e9f767SBen Widawsky } 44721e9f767SBen Widawsky 44821e9f767SBen Widawsky /** 44921e9f767SBen Widawsky * handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace. 45021e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 45121e9f767SBen Widawsky * @cmd: The validated command. 45221e9f767SBen Widawsky * @in_payload: Pointer to userspace's input payload. 45321e9f767SBen Widawsky * @out_payload: Pointer to userspace's output payload. 45421e9f767SBen Widawsky * @size_out: (Input) Max payload size to copy out. 45521e9f767SBen Widawsky * (Output) Payload size hardware generated. 45621e9f767SBen Widawsky * @retval: Hardware generated return code from the operation. 45721e9f767SBen Widawsky * 45821e9f767SBen Widawsky * Return: 45921e9f767SBen Widawsky * * %0 - Mailbox transaction succeeded. This implies the mailbox 46021e9f767SBen Widawsky * protocol completed successfully not that the operation itself 46121e9f767SBen Widawsky * was successful. 46221e9f767SBen Widawsky * * %-ENOMEM - Couldn't allocate a bounce buffer. 46321e9f767SBen Widawsky * * %-EFAULT - Something happened with copy_to/from_user. 46421e9f767SBen Widawsky * * %-EINTR - Mailbox acquisition interrupted. 46521e9f767SBen Widawsky * * %-EXXX - Transaction level failures. 46621e9f767SBen Widawsky * 46721e9f767SBen Widawsky * Creates the appropriate mailbox command and dispatches it on behalf of a 46821e9f767SBen Widawsky * userspace request. The input and output payloads are copied between 46921e9f767SBen Widawsky * userspace. 47021e9f767SBen Widawsky * 47121e9f767SBen Widawsky * See cxl_send_cmd(). 47221e9f767SBen Widawsky */ 47321e9f767SBen Widawsky static int handle_mailbox_cmd_from_user(struct cxl_mem *cxlm, 47421e9f767SBen Widawsky const struct cxl_mem_command *cmd, 47521e9f767SBen Widawsky u64 in_payload, u64 out_payload, 47621e9f767SBen Widawsky s32 *size_out, u32 *retval) 47721e9f767SBen Widawsky { 47821e9f767SBen Widawsky struct device *dev = &cxlm->pdev->dev; 47921e9f767SBen Widawsky struct mbox_cmd mbox_cmd = { 48021e9f767SBen Widawsky .opcode = cmd->opcode, 48121e9f767SBen Widawsky .size_in = cmd->info.size_in, 48221e9f767SBen Widawsky .size_out = cmd->info.size_out, 48321e9f767SBen Widawsky }; 48421e9f767SBen Widawsky int rc; 48521e9f767SBen Widawsky 48621e9f767SBen Widawsky if (cmd->info.size_out) { 48721e9f767SBen Widawsky mbox_cmd.payload_out = kvzalloc(cmd->info.size_out, GFP_KERNEL); 48821e9f767SBen Widawsky if (!mbox_cmd.payload_out) 48921e9f767SBen Widawsky return -ENOMEM; 49021e9f767SBen Widawsky } 49121e9f767SBen Widawsky 49221e9f767SBen Widawsky if (cmd->info.size_in) { 49321e9f767SBen Widawsky mbox_cmd.payload_in = vmemdup_user(u64_to_user_ptr(in_payload), 49421e9f767SBen Widawsky cmd->info.size_in); 49521e9f767SBen Widawsky if (IS_ERR(mbox_cmd.payload_in)) { 49621e9f767SBen Widawsky kvfree(mbox_cmd.payload_out); 49721e9f767SBen Widawsky return PTR_ERR(mbox_cmd.payload_in); 49821e9f767SBen Widawsky } 49921e9f767SBen Widawsky } 50021e9f767SBen Widawsky 50121e9f767SBen Widawsky rc = cxl_mem_mbox_get(cxlm); 50221e9f767SBen Widawsky if (rc) 50321e9f767SBen Widawsky goto out; 50421e9f767SBen Widawsky 50521e9f767SBen Widawsky dev_dbg(dev, 50621e9f767SBen Widawsky "Submitting %s command for user\n" 50721e9f767SBen Widawsky "\topcode: %x\n" 50821e9f767SBen Widawsky "\tsize: %ub\n", 50921e9f767SBen Widawsky cxl_command_names[cmd->info.id].name, mbox_cmd.opcode, 51021e9f767SBen Widawsky cmd->info.size_in); 51121e9f767SBen Widawsky 51221e9f767SBen Widawsky dev_WARN_ONCE(dev, cmd->info.id == CXL_MEM_COMMAND_ID_RAW, 51321e9f767SBen Widawsky "raw command path used\n"); 51421e9f767SBen Widawsky 51521e9f767SBen Widawsky rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd); 51621e9f767SBen Widawsky cxl_mem_mbox_put(cxlm); 51721e9f767SBen Widawsky if (rc) 51821e9f767SBen Widawsky goto out; 51921e9f767SBen Widawsky 52021e9f767SBen Widawsky /* 52121e9f767SBen Widawsky * @size_out contains the max size that's allowed to be written back out 52221e9f767SBen Widawsky * to userspace. While the payload may have written more output than 52321e9f767SBen Widawsky * this it will have to be ignored. 52421e9f767SBen Widawsky */ 52521e9f767SBen Widawsky if (mbox_cmd.size_out) { 52621e9f767SBen Widawsky dev_WARN_ONCE(dev, mbox_cmd.size_out > *size_out, 52721e9f767SBen Widawsky "Invalid return size\n"); 52821e9f767SBen Widawsky if (copy_to_user(u64_to_user_ptr(out_payload), 52921e9f767SBen Widawsky mbox_cmd.payload_out, mbox_cmd.size_out)) { 53021e9f767SBen Widawsky rc = -EFAULT; 53121e9f767SBen Widawsky goto out; 53221e9f767SBen Widawsky } 53321e9f767SBen Widawsky } 53421e9f767SBen Widawsky 53521e9f767SBen Widawsky *size_out = mbox_cmd.size_out; 53621e9f767SBen Widawsky *retval = mbox_cmd.return_code; 53721e9f767SBen Widawsky 53821e9f767SBen Widawsky out: 53921e9f767SBen Widawsky kvfree(mbox_cmd.payload_in); 54021e9f767SBen Widawsky kvfree(mbox_cmd.payload_out); 54121e9f767SBen Widawsky return rc; 54221e9f767SBen Widawsky } 54321e9f767SBen Widawsky 54421e9f767SBen Widawsky static bool cxl_mem_raw_command_allowed(u16 opcode) 54521e9f767SBen Widawsky { 54621e9f767SBen Widawsky int i; 54721e9f767SBen Widawsky 54821e9f767SBen Widawsky if (!IS_ENABLED(CONFIG_CXL_MEM_RAW_COMMANDS)) 54921e9f767SBen Widawsky return false; 55021e9f767SBen Widawsky 55121e9f767SBen Widawsky if (security_locked_down(LOCKDOWN_NONE)) 55221e9f767SBen Widawsky return false; 55321e9f767SBen Widawsky 55421e9f767SBen Widawsky if (cxl_raw_allow_all) 55521e9f767SBen Widawsky return true; 55621e9f767SBen Widawsky 55721e9f767SBen Widawsky if (cxl_is_security_command(opcode)) 55821e9f767SBen Widawsky return false; 55921e9f767SBen Widawsky 56021e9f767SBen Widawsky for (i = 0; i < ARRAY_SIZE(cxl_disabled_raw_commands); i++) 56121e9f767SBen Widawsky if (cxl_disabled_raw_commands[i] == opcode) 56221e9f767SBen Widawsky return false; 56321e9f767SBen Widawsky 56421e9f767SBen Widawsky return true; 56521e9f767SBen Widawsky } 56621e9f767SBen Widawsky 56721e9f767SBen Widawsky /** 56821e9f767SBen Widawsky * cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND. 56921e9f767SBen Widawsky * @cxlm: &struct cxl_mem device whose mailbox will be used. 57021e9f767SBen Widawsky * @send_cmd: &struct cxl_send_command copied in from userspace. 57121e9f767SBen Widawsky * @out_cmd: Sanitized and populated &struct cxl_mem_command. 57221e9f767SBen Widawsky * 57321e9f767SBen Widawsky * Return: 57421e9f767SBen Widawsky * * %0 - @out_cmd is ready to send. 57521e9f767SBen Widawsky * * %-ENOTTY - Invalid command specified. 57621e9f767SBen Widawsky * * %-EINVAL - Reserved fields or invalid values were used. 57721e9f767SBen Widawsky * * %-ENOMEM - Input or output buffer wasn't sized properly. 57821e9f767SBen Widawsky * * %-EPERM - Attempted to use a protected command. 57921e9f767SBen Widawsky * 58021e9f767SBen Widawsky * The result of this command is a fully validated command in @out_cmd that is 58121e9f767SBen Widawsky * safe to send to the hardware. 58221e9f767SBen Widawsky * 58321e9f767SBen Widawsky * See handle_mailbox_cmd_from_user() 58421e9f767SBen Widawsky */ 58521e9f767SBen Widawsky static int cxl_validate_cmd_from_user(struct cxl_mem *cxlm, 58621e9f767SBen Widawsky const struct cxl_send_command *send_cmd, 58721e9f767SBen Widawsky struct cxl_mem_command *out_cmd) 58821e9f767SBen Widawsky { 58921e9f767SBen Widawsky const struct cxl_command_info *info; 59021e9f767SBen Widawsky struct cxl_mem_command *c; 59121e9f767SBen Widawsky 59221e9f767SBen Widawsky if (send_cmd->id == 0 || send_cmd->id >= CXL_MEM_COMMAND_ID_MAX) 59321e9f767SBen Widawsky return -ENOTTY; 59421e9f767SBen Widawsky 59521e9f767SBen Widawsky /* 59621e9f767SBen Widawsky * The user can never specify an input payload larger than what hardware 59721e9f767SBen Widawsky * supports, but output can be arbitrarily large (simply write out as 59821e9f767SBen Widawsky * much data as the hardware provides). 59921e9f767SBen Widawsky */ 60021e9f767SBen Widawsky if (send_cmd->in.size > cxlm->payload_size) 60121e9f767SBen Widawsky return -EINVAL; 60221e9f767SBen Widawsky 60321e9f767SBen Widawsky /* 60421e9f767SBen Widawsky * Checks are bypassed for raw commands but a WARN/taint will occur 60521e9f767SBen Widawsky * later in the callchain 60621e9f767SBen Widawsky */ 60721e9f767SBen Widawsky if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW) { 60821e9f767SBen Widawsky const struct cxl_mem_command temp = { 60921e9f767SBen Widawsky .info = { 61021e9f767SBen Widawsky .id = CXL_MEM_COMMAND_ID_RAW, 61121e9f767SBen Widawsky .flags = 0, 61221e9f767SBen Widawsky .size_in = send_cmd->in.size, 61321e9f767SBen Widawsky .size_out = send_cmd->out.size, 61421e9f767SBen Widawsky }, 61521e9f767SBen Widawsky .opcode = send_cmd->raw.opcode 61621e9f767SBen Widawsky }; 61721e9f767SBen Widawsky 61821e9f767SBen Widawsky if (send_cmd->raw.rsvd) 61921e9f767SBen Widawsky return -EINVAL; 62021e9f767SBen Widawsky 62121e9f767SBen Widawsky /* 62221e9f767SBen Widawsky * Unlike supported commands, the output size of RAW commands 62321e9f767SBen Widawsky * gets passed along without further checking, so it must be 62421e9f767SBen Widawsky * validated here. 62521e9f767SBen Widawsky */ 62621e9f767SBen Widawsky if (send_cmd->out.size > cxlm->payload_size) 62721e9f767SBen Widawsky return -EINVAL; 62821e9f767SBen Widawsky 62921e9f767SBen Widawsky if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode)) 63021e9f767SBen Widawsky return -EPERM; 63121e9f767SBen Widawsky 63221e9f767SBen Widawsky memcpy(out_cmd, &temp, sizeof(temp)); 63321e9f767SBen Widawsky 63421e9f767SBen Widawsky return 0; 63521e9f767SBen Widawsky } 63621e9f767SBen Widawsky 63721e9f767SBen Widawsky if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK) 63821e9f767SBen Widawsky return -EINVAL; 63921e9f767SBen Widawsky 64021e9f767SBen Widawsky if (send_cmd->rsvd) 64121e9f767SBen Widawsky return -EINVAL; 64221e9f767SBen Widawsky 64321e9f767SBen Widawsky if (send_cmd->in.rsvd || send_cmd->out.rsvd) 64421e9f767SBen Widawsky return -EINVAL; 64521e9f767SBen Widawsky 64621e9f767SBen Widawsky /* Convert user's command into the internal representation */ 64721e9f767SBen Widawsky c = &mem_commands[send_cmd->id]; 64821e9f767SBen Widawsky info = &c->info; 64921e9f767SBen Widawsky 65021e9f767SBen Widawsky /* Check that the command is enabled for hardware */ 65121e9f767SBen Widawsky if (!test_bit(info->id, cxlm->enabled_cmds)) 65221e9f767SBen Widawsky return -ENOTTY; 65321e9f767SBen Widawsky 65421e9f767SBen Widawsky /* Check the input buffer is the expected size */ 65521e9f767SBen Widawsky if (info->size_in >= 0 && info->size_in != send_cmd->in.size) 65621e9f767SBen Widawsky return -ENOMEM; 65721e9f767SBen Widawsky 65821e9f767SBen Widawsky /* Check the output buffer is at least large enough */ 65921e9f767SBen Widawsky if (info->size_out >= 0 && send_cmd->out.size < info->size_out) 66021e9f767SBen Widawsky return -ENOMEM; 66121e9f767SBen Widawsky 66221e9f767SBen Widawsky memcpy(out_cmd, c, sizeof(*c)); 66321e9f767SBen Widawsky out_cmd->info.size_in = send_cmd->in.size; 66421e9f767SBen Widawsky /* 66521e9f767SBen Widawsky * XXX: out_cmd->info.size_out will be controlled by the driver, and the 66621e9f767SBen Widawsky * specified number of bytes @send_cmd->out.size will be copied back out 66721e9f767SBen Widawsky * to userspace. 66821e9f767SBen Widawsky */ 66921e9f767SBen Widawsky 67021e9f767SBen Widawsky return 0; 67121e9f767SBen Widawsky } 67221e9f767SBen Widawsky 67321e9f767SBen Widawsky static int cxl_query_cmd(struct cxl_memdev *cxlmd, 67421e9f767SBen Widawsky struct cxl_mem_query_commands __user *q) 67521e9f767SBen Widawsky { 67621e9f767SBen Widawsky struct device *dev = &cxlmd->dev; 67721e9f767SBen Widawsky struct cxl_mem_command *cmd; 67821e9f767SBen Widawsky u32 n_commands; 67921e9f767SBen Widawsky int j = 0; 68021e9f767SBen Widawsky 68121e9f767SBen Widawsky dev_dbg(dev, "Query IOCTL\n"); 68221e9f767SBen Widawsky 68321e9f767SBen Widawsky if (get_user(n_commands, &q->n_commands)) 68421e9f767SBen Widawsky return -EFAULT; 68521e9f767SBen Widawsky 68621e9f767SBen Widawsky /* returns the total number if 0 elements are requested. */ 68721e9f767SBen Widawsky if (n_commands == 0) 68821e9f767SBen Widawsky return put_user(cxl_cmd_count, &q->n_commands); 68921e9f767SBen Widawsky 69021e9f767SBen Widawsky /* 69121e9f767SBen Widawsky * otherwise, return max(n_commands, total commands) cxl_command_info 69221e9f767SBen Widawsky * structures. 69321e9f767SBen Widawsky */ 69421e9f767SBen Widawsky cxl_for_each_cmd(cmd) { 69521e9f767SBen Widawsky const struct cxl_command_info *info = &cmd->info; 69621e9f767SBen Widawsky 69721e9f767SBen Widawsky if (copy_to_user(&q->commands[j++], info, sizeof(*info))) 69821e9f767SBen Widawsky return -EFAULT; 69921e9f767SBen Widawsky 70021e9f767SBen Widawsky if (j == n_commands) 70121e9f767SBen Widawsky break; 70221e9f767SBen Widawsky } 70321e9f767SBen Widawsky 70421e9f767SBen Widawsky return 0; 70521e9f767SBen Widawsky } 70621e9f767SBen Widawsky 70721e9f767SBen Widawsky static int cxl_send_cmd(struct cxl_memdev *cxlmd, 70821e9f767SBen Widawsky struct cxl_send_command __user *s) 70921e9f767SBen Widawsky { 71021e9f767SBen Widawsky struct cxl_mem *cxlm = cxlmd->cxlm; 71121e9f767SBen Widawsky struct device *dev = &cxlmd->dev; 71221e9f767SBen Widawsky struct cxl_send_command send; 71321e9f767SBen Widawsky struct cxl_mem_command c; 71421e9f767SBen Widawsky int rc; 71521e9f767SBen Widawsky 71621e9f767SBen Widawsky dev_dbg(dev, "Send IOCTL\n"); 71721e9f767SBen Widawsky 71821e9f767SBen Widawsky if (copy_from_user(&send, s, sizeof(send))) 71921e9f767SBen Widawsky return -EFAULT; 72021e9f767SBen Widawsky 72121e9f767SBen Widawsky rc = cxl_validate_cmd_from_user(cxlmd->cxlm, &send, &c); 72221e9f767SBen Widawsky if (rc) 72321e9f767SBen Widawsky return rc; 72421e9f767SBen Widawsky 72521e9f767SBen Widawsky /* Prepare to handle a full payload for variable sized output */ 72621e9f767SBen Widawsky if (c.info.size_out < 0) 72721e9f767SBen Widawsky c.info.size_out = cxlm->payload_size; 72821e9f767SBen Widawsky 72921e9f767SBen Widawsky rc = handle_mailbox_cmd_from_user(cxlm, &c, send.in.payload, 73021e9f767SBen Widawsky send.out.payload, &send.out.size, 73121e9f767SBen Widawsky &send.retval); 73221e9f767SBen Widawsky if (rc) 73321e9f767SBen Widawsky return rc; 73421e9f767SBen Widawsky 73521e9f767SBen Widawsky if (copy_to_user(s, &send, sizeof(send))) 73621e9f767SBen Widawsky return -EFAULT; 73721e9f767SBen Widawsky 73821e9f767SBen Widawsky return 0; 73921e9f767SBen Widawsky } 74021e9f767SBen Widawsky 74121e9f767SBen Widawsky static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, 74221e9f767SBen Widawsky unsigned long arg) 74321e9f767SBen Widawsky { 74421e9f767SBen Widawsky switch (cmd) { 74521e9f767SBen Widawsky case CXL_MEM_QUERY_COMMANDS: 74621e9f767SBen Widawsky return cxl_query_cmd(cxlmd, (void __user *)arg); 74721e9f767SBen Widawsky case CXL_MEM_SEND_COMMAND: 74821e9f767SBen Widawsky return cxl_send_cmd(cxlmd, (void __user *)arg); 74921e9f767SBen Widawsky default: 75021e9f767SBen Widawsky return -ENOTTY; 75121e9f767SBen Widawsky } 75221e9f767SBen Widawsky } 75321e9f767SBen Widawsky 75421e9f767SBen Widawsky static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, 75521e9f767SBen Widawsky unsigned long arg) 75621e9f767SBen Widawsky { 75721e9f767SBen Widawsky struct cxl_memdev *cxlmd = file->private_data; 75821e9f767SBen Widawsky int rc = -ENXIO; 75921e9f767SBen Widawsky 76021e9f767SBen Widawsky down_read(&cxl_memdev_rwsem); 76121e9f767SBen Widawsky if (cxlmd->cxlm) 76221e9f767SBen Widawsky rc = __cxl_memdev_ioctl(cxlmd, cmd, arg); 76321e9f767SBen Widawsky up_read(&cxl_memdev_rwsem); 76421e9f767SBen Widawsky 76521e9f767SBen Widawsky return rc; 76621e9f767SBen Widawsky } 76721e9f767SBen Widawsky 76821e9f767SBen Widawsky static int cxl_memdev_open(struct inode *inode, struct file *file) 76921e9f767SBen Widawsky { 77021e9f767SBen Widawsky struct cxl_memdev *cxlmd = 77121e9f767SBen Widawsky container_of(inode->i_cdev, typeof(*cxlmd), cdev); 77221e9f767SBen Widawsky 77321e9f767SBen Widawsky get_device(&cxlmd->dev); 77421e9f767SBen Widawsky file->private_data = cxlmd; 77521e9f767SBen Widawsky 77621e9f767SBen Widawsky return 0; 77721e9f767SBen Widawsky } 77821e9f767SBen Widawsky 77921e9f767SBen Widawsky static int cxl_memdev_release_file(struct inode *inode, struct file *file) 78021e9f767SBen Widawsky { 78121e9f767SBen Widawsky struct cxl_memdev *cxlmd = 78221e9f767SBen Widawsky container_of(inode->i_cdev, typeof(*cxlmd), cdev); 78321e9f767SBen Widawsky 78421e9f767SBen Widawsky put_device(&cxlmd->dev); 78521e9f767SBen Widawsky 78621e9f767SBen Widawsky return 0; 78721e9f767SBen Widawsky } 78821e9f767SBen Widawsky 78921e9f767SBen Widawsky static const struct file_operations cxl_memdev_fops = { 79021e9f767SBen Widawsky .owner = THIS_MODULE, 79121e9f767SBen Widawsky .unlocked_ioctl = cxl_memdev_ioctl, 79221e9f767SBen Widawsky .open = cxl_memdev_open, 79321e9f767SBen Widawsky .release = cxl_memdev_release_file, 79421e9f767SBen Widawsky .compat_ioctl = compat_ptr_ioctl, 79521e9f767SBen Widawsky .llseek = noop_llseek, 79621e9f767SBen Widawsky }; 79721e9f767SBen Widawsky 79821e9f767SBen Widawsky static inline struct cxl_mem_command *cxl_mem_find_command(u16 opcode) 79921e9f767SBen Widawsky { 80021e9f767SBen Widawsky struct cxl_mem_command *c; 80121e9f767SBen Widawsky 80221e9f767SBen Widawsky cxl_for_each_cmd(c) 80321e9f767SBen Widawsky if (c->opcode == opcode) 80421e9f767SBen Widawsky return c; 80521e9f767SBen Widawsky 80621e9f767SBen Widawsky return NULL; 80721e9f767SBen Widawsky } 80821e9f767SBen Widawsky 80921e9f767SBen Widawsky /** 81021e9f767SBen Widawsky * cxl_mem_mbox_send_cmd() - Send a mailbox command to a memory device. 81121e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 81221e9f767SBen Widawsky * @opcode: Opcode for the mailbox command. 81321e9f767SBen Widawsky * @in: The input payload for the mailbox command. 81421e9f767SBen Widawsky * @in_size: The length of the input payload 81521e9f767SBen Widawsky * @out: Caller allocated buffer for the output. 81621e9f767SBen Widawsky * @out_size: Expected size of output. 81721e9f767SBen Widawsky * 81821e9f767SBen Widawsky * Context: Any context. Will acquire and release mbox_mutex. 81921e9f767SBen Widawsky * Return: 82021e9f767SBen Widawsky * * %>=0 - Number of bytes returned in @out. 82121e9f767SBen Widawsky * * %-E2BIG - Payload is too large for hardware. 82221e9f767SBen Widawsky * * %-EBUSY - Couldn't acquire exclusive mailbox access. 82321e9f767SBen Widawsky * * %-EFAULT - Hardware error occurred. 82421e9f767SBen Widawsky * * %-ENXIO - Command completed, but device reported an error. 82521e9f767SBen Widawsky * * %-EIO - Unexpected output size. 82621e9f767SBen Widawsky * 82721e9f767SBen Widawsky * Mailbox commands may execute successfully yet the device itself reported an 82821e9f767SBen Widawsky * error. While this distinction can be useful for commands from userspace, the 82921e9f767SBen Widawsky * kernel will only be able to use results when both are successful. 83021e9f767SBen Widawsky * 83121e9f767SBen Widawsky * See __cxl_mem_mbox_send_cmd() 83221e9f767SBen Widawsky */ 83321e9f767SBen Widawsky static int cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, u16 opcode, 83421e9f767SBen Widawsky void *in, size_t in_size, 83521e9f767SBen Widawsky void *out, size_t out_size) 83621e9f767SBen Widawsky { 83721e9f767SBen Widawsky const struct cxl_mem_command *cmd = cxl_mem_find_command(opcode); 83821e9f767SBen Widawsky struct mbox_cmd mbox_cmd = { 83921e9f767SBen Widawsky .opcode = opcode, 84021e9f767SBen Widawsky .payload_in = in, 84121e9f767SBen Widawsky .size_in = in_size, 84221e9f767SBen Widawsky .size_out = out_size, 84321e9f767SBen Widawsky .payload_out = out, 84421e9f767SBen Widawsky }; 84521e9f767SBen Widawsky int rc; 84621e9f767SBen Widawsky 84721e9f767SBen Widawsky if (out_size > cxlm->payload_size) 84821e9f767SBen Widawsky return -E2BIG; 84921e9f767SBen Widawsky 85021e9f767SBen Widawsky rc = cxl_mem_mbox_get(cxlm); 85121e9f767SBen Widawsky if (rc) 85221e9f767SBen Widawsky return rc; 85321e9f767SBen Widawsky 85421e9f767SBen Widawsky rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd); 85521e9f767SBen Widawsky cxl_mem_mbox_put(cxlm); 85621e9f767SBen Widawsky if (rc) 85721e9f767SBen Widawsky return rc; 85821e9f767SBen Widawsky 85921e9f767SBen Widawsky /* TODO: Map return code to proper kernel style errno */ 86021e9f767SBen Widawsky if (mbox_cmd.return_code != CXL_MBOX_SUCCESS) 86121e9f767SBen Widawsky return -ENXIO; 86221e9f767SBen Widawsky 86321e9f767SBen Widawsky /* 86421e9f767SBen Widawsky * Variable sized commands can't be validated and so it's up to the 86521e9f767SBen Widawsky * caller to do that if they wish. 86621e9f767SBen Widawsky */ 86721e9f767SBen Widawsky if (cmd->info.size_out >= 0 && mbox_cmd.size_out != out_size) 86821e9f767SBen Widawsky return -EIO; 86921e9f767SBen Widawsky 87021e9f767SBen Widawsky return 0; 87121e9f767SBen Widawsky } 87221e9f767SBen Widawsky 87321e9f767SBen Widawsky static int cxl_mem_setup_mailbox(struct cxl_mem *cxlm) 87421e9f767SBen Widawsky { 87521e9f767SBen Widawsky const int cap = readl(cxlm->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET); 87621e9f767SBen Widawsky 87721e9f767SBen Widawsky cxlm->payload_size = 87821e9f767SBen Widawsky 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap); 87921e9f767SBen Widawsky 88021e9f767SBen Widawsky /* 88121e9f767SBen Widawsky * CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register 88221e9f767SBen Widawsky * 88321e9f767SBen Widawsky * If the size is too small, mandatory commands will not work and so 88421e9f767SBen Widawsky * there's no point in going forward. If the size is too large, there's 88521e9f767SBen Widawsky * no harm is soft limiting it. 88621e9f767SBen Widawsky */ 88721e9f767SBen Widawsky cxlm->payload_size = min_t(size_t, cxlm->payload_size, SZ_1M); 88821e9f767SBen Widawsky if (cxlm->payload_size < 256) { 88921e9f767SBen Widawsky dev_err(&cxlm->pdev->dev, "Mailbox is too small (%zub)", 89021e9f767SBen Widawsky cxlm->payload_size); 89121e9f767SBen Widawsky return -ENXIO; 89221e9f767SBen Widawsky } 89321e9f767SBen Widawsky 89421e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, "Mailbox payload sized %zu", 89521e9f767SBen Widawsky cxlm->payload_size); 89621e9f767SBen Widawsky 89721e9f767SBen Widawsky return 0; 89821e9f767SBen Widawsky } 89921e9f767SBen Widawsky 9001b0a1a2aSBen Widawsky static struct cxl_mem *cxl_mem_create(struct pci_dev *pdev) 90121e9f767SBen Widawsky { 90221e9f767SBen Widawsky struct device *dev = &pdev->dev; 90321e9f767SBen Widawsky struct cxl_mem *cxlm; 90421e9f767SBen Widawsky 9055d0c6f02SBen Widawsky cxlm = devm_kzalloc(dev, sizeof(*cxlm), GFP_KERNEL); 90621e9f767SBen Widawsky if (!cxlm) { 90721e9f767SBen Widawsky dev_err(dev, "No memory available\n"); 9081b0a1a2aSBen Widawsky return ERR_PTR(-ENOMEM); 90921e9f767SBen Widawsky } 91021e9f767SBen Widawsky 9111b0a1a2aSBen Widawsky mutex_init(&cxlm->mbox_mutex); 9121b0a1a2aSBen Widawsky cxlm->pdev = pdev; 9131b0a1a2aSBen Widawsky cxlm->enabled_cmds = 9141b0a1a2aSBen Widawsky devm_kmalloc_array(dev, BITS_TO_LONGS(cxl_cmd_count), 9151b0a1a2aSBen Widawsky sizeof(unsigned long), 9161b0a1a2aSBen Widawsky GFP_KERNEL | __GFP_ZERO); 9171b0a1a2aSBen Widawsky if (!cxlm->enabled_cmds) { 9181b0a1a2aSBen Widawsky dev_err(dev, "No memory available for bitmap\n"); 9191b0a1a2aSBen Widawsky return ERR_PTR(-ENOMEM); 9201b0a1a2aSBen Widawsky } 9211b0a1a2aSBen Widawsky 9221b0a1a2aSBen Widawsky return cxlm; 9231b0a1a2aSBen Widawsky } 9241b0a1a2aSBen Widawsky 925*07d62eacSIra Weiny static void __iomem *cxl_mem_map_regblock(struct cxl_mem *cxlm, 926*07d62eacSIra Weiny u8 bar, u64 offset) 9271b0a1a2aSBen Widawsky { 9281b0a1a2aSBen Widawsky struct pci_dev *pdev = cxlm->pdev; 9291b0a1a2aSBen Widawsky struct device *dev = &pdev->dev; 9301b0a1a2aSBen Widawsky int rc; 9311b0a1a2aSBen Widawsky 93221e9f767SBen Widawsky /* Basic sanity check that BAR is big enough */ 93321e9f767SBen Widawsky if (pci_resource_len(pdev, bar) < offset) { 93421e9f767SBen Widawsky dev_err(dev, "BAR%d: %pr: too small (offset: %#llx)\n", bar, 93521e9f767SBen Widawsky &pdev->resource[bar], (unsigned long long)offset); 9366630d31cSBen Widawsky return IOMEM_ERR_PTR(-ENXIO); 93721e9f767SBen Widawsky } 93821e9f767SBen Widawsky 93921e9f767SBen Widawsky rc = pcim_iomap_regions(pdev, BIT(bar), pci_name(pdev)); 94021e9f767SBen Widawsky if (rc) { 94121e9f767SBen Widawsky dev_err(dev, "failed to map registers\n"); 9426630d31cSBen Widawsky return IOMEM_ERR_PTR(rc); 94321e9f767SBen Widawsky } 94421e9f767SBen Widawsky 94521e9f767SBen Widawsky dev_dbg(dev, "Mapped CXL Memory Device resource\n"); 9466630d31cSBen Widawsky 9476630d31cSBen Widawsky return pcim_iomap_table(pdev)[bar] + offset; 94821e9f767SBen Widawsky } 94921e9f767SBen Widawsky 95021e9f767SBen Widawsky static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec) 95121e9f767SBen Widawsky { 95221e9f767SBen Widawsky int pos; 95321e9f767SBen Widawsky 95421e9f767SBen Widawsky pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DVSEC); 95521e9f767SBen Widawsky if (!pos) 95621e9f767SBen Widawsky return 0; 95721e9f767SBen Widawsky 95821e9f767SBen Widawsky while (pos) { 95921e9f767SBen Widawsky u16 vendor, id; 96021e9f767SBen Widawsky 96121e9f767SBen Widawsky pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER1, &vendor); 96221e9f767SBen Widawsky pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER2, &id); 96321e9f767SBen Widawsky if (vendor == PCI_DVSEC_VENDOR_ID_CXL && dvsec == id) 96421e9f767SBen Widawsky return pos; 96521e9f767SBen Widawsky 96621e9f767SBen Widawsky pos = pci_find_next_ext_capability(pdev, pos, 96721e9f767SBen Widawsky PCI_EXT_CAP_ID_DVSEC); 96821e9f767SBen Widawsky } 96921e9f767SBen Widawsky 97021e9f767SBen Widawsky return 0; 97121e9f767SBen Widawsky } 97221e9f767SBen Widawsky 973*07d62eacSIra Weiny static void cxl_decode_register_block(u32 reg_lo, u32 reg_hi, 974*07d62eacSIra Weiny u8 *bar, u64 *offset, u8 *reg_type) 975*07d62eacSIra Weiny { 976*07d62eacSIra Weiny *offset = ((u64)reg_hi << 32) | (reg_lo & CXL_REGLOC_ADDR_MASK); 977*07d62eacSIra Weiny *bar = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo); 978*07d62eacSIra Weiny *reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); 979*07d62eacSIra Weiny } 980*07d62eacSIra Weiny 9811d5a4159SBen Widawsky /** 9821d5a4159SBen Widawsky * cxl_mem_setup_regs() - Setup necessary MMIO. 9831d5a4159SBen Widawsky * @cxlm: The CXL memory device to communicate with. 9841d5a4159SBen Widawsky * 9851d5a4159SBen Widawsky * Return: 0 if all necessary registers mapped. 9861d5a4159SBen Widawsky * 9871d5a4159SBen Widawsky * A memory device is required by spec to implement a certain set of MMIO 9881d5a4159SBen Widawsky * regions. The purpose of this function is to enumerate and map those 9891d5a4159SBen Widawsky * registers. 9901d5a4159SBen Widawsky */ 9911d5a4159SBen Widawsky static int cxl_mem_setup_regs(struct cxl_mem *cxlm) 9921d5a4159SBen Widawsky { 9931d5a4159SBen Widawsky struct cxl_regs *regs = &cxlm->regs; 9941d5a4159SBen Widawsky struct pci_dev *pdev = cxlm->pdev; 9951d5a4159SBen Widawsky struct device *dev = &pdev->dev; 9961d5a4159SBen Widawsky u32 regloc_size, regblocks; 9976630d31cSBen Widawsky void __iomem *base; 9986630d31cSBen Widawsky int regloc, i; 9991d5a4159SBen Widawsky 10001d5a4159SBen Widawsky regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_OFFSET); 10011d5a4159SBen Widawsky if (!regloc) { 10021d5a4159SBen Widawsky dev_err(dev, "register location dvsec not found\n"); 10031d5a4159SBen Widawsky return -ENXIO; 10041d5a4159SBen Widawsky } 10051d5a4159SBen Widawsky 10061d5a4159SBen Widawsky /* Get the size of the Register Locator DVSEC */ 10071d5a4159SBen Widawsky pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size); 10081d5a4159SBen Widawsky regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size); 10091d5a4159SBen Widawsky 10101d5a4159SBen Widawsky regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET; 10111d5a4159SBen Widawsky regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8; 10121d5a4159SBen Widawsky 10131d5a4159SBen Widawsky for (i = 0; i < regblocks; i++, regloc += 8) { 10141d5a4159SBen Widawsky u32 reg_lo, reg_hi; 10151d5a4159SBen Widawsky u8 reg_type; 1016*07d62eacSIra Weiny u64 offset; 1017*07d62eacSIra Weiny u8 bar; 10181d5a4159SBen Widawsky 10191d5a4159SBen Widawsky /* "register low and high" contain other bits */ 10201d5a4159SBen Widawsky pci_read_config_dword(pdev, regloc, ®_lo); 10211d5a4159SBen Widawsky pci_read_config_dword(pdev, regloc + 4, ®_hi); 10221d5a4159SBen Widawsky 1023*07d62eacSIra Weiny cxl_decode_register_block(reg_lo, reg_hi, &bar, &offset, 1024*07d62eacSIra Weiny ®_type); 1025*07d62eacSIra Weiny 1026*07d62eacSIra Weiny dev_dbg(dev, "Found register block in bar %u @ 0x%llx of type %u\n", 1027*07d62eacSIra Weiny bar, offset, reg_type); 10281d5a4159SBen Widawsky 10291d5a4159SBen Widawsky if (reg_type == CXL_REGLOC_RBI_MEMDEV) { 1030*07d62eacSIra Weiny base = cxl_mem_map_regblock(cxlm, bar, offset); 10316630d31cSBen Widawsky if (IS_ERR(base)) 10326630d31cSBen Widawsky return PTR_ERR(base); 10331d5a4159SBen Widawsky break; 10341d5a4159SBen Widawsky } 10351d5a4159SBen Widawsky } 10361d5a4159SBen Widawsky 10371d5a4159SBen Widawsky if (i == regblocks) { 10381d5a4159SBen Widawsky dev_err(dev, "Missing register locator for device registers\n"); 10391d5a4159SBen Widawsky return -ENXIO; 10401d5a4159SBen Widawsky } 10411d5a4159SBen Widawsky 10426630d31cSBen Widawsky cxl_setup_device_regs(dev, base, ®s->device_regs); 10431d5a4159SBen Widawsky 10441d5a4159SBen Widawsky if (!regs->status || !regs->mbox || !regs->memdev) { 10451d5a4159SBen Widawsky dev_err(dev, "registers not found: %s%s%s\n", 10461d5a4159SBen Widawsky !regs->status ? "status " : "", 10471d5a4159SBen Widawsky !regs->mbox ? "mbox " : "", 10481d5a4159SBen Widawsky !regs->memdev ? "memdev" : ""); 10491d5a4159SBen Widawsky return -ENXIO; 10501d5a4159SBen Widawsky } 10511d5a4159SBen Widawsky 10521d5a4159SBen Widawsky return 0; 10531d5a4159SBen Widawsky } 10541d5a4159SBen Widawsky 105521e9f767SBen Widawsky static struct cxl_memdev *to_cxl_memdev(struct device *dev) 105621e9f767SBen Widawsky { 105721e9f767SBen Widawsky return container_of(dev, struct cxl_memdev, dev); 105821e9f767SBen Widawsky } 105921e9f767SBen Widawsky 106021e9f767SBen Widawsky static void cxl_memdev_release(struct device *dev) 106121e9f767SBen Widawsky { 106221e9f767SBen Widawsky struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 106321e9f767SBen Widawsky 106421e9f767SBen Widawsky ida_free(&cxl_memdev_ida, cxlmd->id); 106521e9f767SBen Widawsky kfree(cxlmd); 106621e9f767SBen Widawsky } 106721e9f767SBen Widawsky 106821e9f767SBen Widawsky static char *cxl_memdev_devnode(struct device *dev, umode_t *mode, kuid_t *uid, 106921e9f767SBen Widawsky kgid_t *gid) 107021e9f767SBen Widawsky { 107121e9f767SBen Widawsky return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); 107221e9f767SBen Widawsky } 107321e9f767SBen Widawsky 107421e9f767SBen Widawsky static ssize_t firmware_version_show(struct device *dev, 107521e9f767SBen Widawsky struct device_attribute *attr, char *buf) 107621e9f767SBen Widawsky { 107721e9f767SBen Widawsky struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 107821e9f767SBen Widawsky struct cxl_mem *cxlm = cxlmd->cxlm; 107921e9f767SBen Widawsky 108021e9f767SBen Widawsky return sysfs_emit(buf, "%.16s\n", cxlm->firmware_version); 108121e9f767SBen Widawsky } 108221e9f767SBen Widawsky static DEVICE_ATTR_RO(firmware_version); 108321e9f767SBen Widawsky 108421e9f767SBen Widawsky static ssize_t payload_max_show(struct device *dev, 108521e9f767SBen Widawsky struct device_attribute *attr, char *buf) 108621e9f767SBen Widawsky { 108721e9f767SBen Widawsky struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 108821e9f767SBen Widawsky struct cxl_mem *cxlm = cxlmd->cxlm; 108921e9f767SBen Widawsky 109021e9f767SBen Widawsky return sysfs_emit(buf, "%zu\n", cxlm->payload_size); 109121e9f767SBen Widawsky } 109221e9f767SBen Widawsky static DEVICE_ATTR_RO(payload_max); 109321e9f767SBen Widawsky 1094199cf8c3SVishal Verma static ssize_t label_storage_size_show(struct device *dev, 1095199cf8c3SVishal Verma struct device_attribute *attr, char *buf) 1096199cf8c3SVishal Verma { 1097199cf8c3SVishal Verma struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 1098199cf8c3SVishal Verma struct cxl_mem *cxlm = cxlmd->cxlm; 1099199cf8c3SVishal Verma 1100199cf8c3SVishal Verma return sysfs_emit(buf, "%zu\n", cxlm->lsa_size); 1101199cf8c3SVishal Verma } 1102199cf8c3SVishal Verma static DEVICE_ATTR_RO(label_storage_size); 1103199cf8c3SVishal Verma 110421e9f767SBen Widawsky static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, 110521e9f767SBen Widawsky char *buf) 110621e9f767SBen Widawsky { 110721e9f767SBen Widawsky struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 110821e9f767SBen Widawsky struct cxl_mem *cxlm = cxlmd->cxlm; 110921e9f767SBen Widawsky unsigned long long len = range_len(&cxlm->ram_range); 111021e9f767SBen Widawsky 111121e9f767SBen Widawsky return sysfs_emit(buf, "%#llx\n", len); 111221e9f767SBen Widawsky } 111321e9f767SBen Widawsky 111421e9f767SBen Widawsky static struct device_attribute dev_attr_ram_size = 111521e9f767SBen Widawsky __ATTR(size, 0444, ram_size_show, NULL); 111621e9f767SBen Widawsky 111721e9f767SBen Widawsky static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, 111821e9f767SBen Widawsky char *buf) 111921e9f767SBen Widawsky { 112021e9f767SBen Widawsky struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 112121e9f767SBen Widawsky struct cxl_mem *cxlm = cxlmd->cxlm; 112221e9f767SBen Widawsky unsigned long long len = range_len(&cxlm->pmem_range); 112321e9f767SBen Widawsky 112421e9f767SBen Widawsky return sysfs_emit(buf, "%#llx\n", len); 112521e9f767SBen Widawsky } 112621e9f767SBen Widawsky 112721e9f767SBen Widawsky static struct device_attribute dev_attr_pmem_size = 112821e9f767SBen Widawsky __ATTR(size, 0444, pmem_size_show, NULL); 112921e9f767SBen Widawsky 113021e9f767SBen Widawsky static struct attribute *cxl_memdev_attributes[] = { 113121e9f767SBen Widawsky &dev_attr_firmware_version.attr, 113221e9f767SBen Widawsky &dev_attr_payload_max.attr, 1133199cf8c3SVishal Verma &dev_attr_label_storage_size.attr, 113421e9f767SBen Widawsky NULL, 113521e9f767SBen Widawsky }; 113621e9f767SBen Widawsky 113721e9f767SBen Widawsky static struct attribute *cxl_memdev_pmem_attributes[] = { 113821e9f767SBen Widawsky &dev_attr_pmem_size.attr, 113921e9f767SBen Widawsky NULL, 114021e9f767SBen Widawsky }; 114121e9f767SBen Widawsky 114221e9f767SBen Widawsky static struct attribute *cxl_memdev_ram_attributes[] = { 114321e9f767SBen Widawsky &dev_attr_ram_size.attr, 114421e9f767SBen Widawsky NULL, 114521e9f767SBen Widawsky }; 114621e9f767SBen Widawsky 114721e9f767SBen Widawsky static struct attribute_group cxl_memdev_attribute_group = { 114821e9f767SBen Widawsky .attrs = cxl_memdev_attributes, 114921e9f767SBen Widawsky }; 115021e9f767SBen Widawsky 115121e9f767SBen Widawsky static struct attribute_group cxl_memdev_ram_attribute_group = { 115221e9f767SBen Widawsky .name = "ram", 115321e9f767SBen Widawsky .attrs = cxl_memdev_ram_attributes, 115421e9f767SBen Widawsky }; 115521e9f767SBen Widawsky 115621e9f767SBen Widawsky static struct attribute_group cxl_memdev_pmem_attribute_group = { 115721e9f767SBen Widawsky .name = "pmem", 115821e9f767SBen Widawsky .attrs = cxl_memdev_pmem_attributes, 115921e9f767SBen Widawsky }; 116021e9f767SBen Widawsky 116121e9f767SBen Widawsky static const struct attribute_group *cxl_memdev_attribute_groups[] = { 116221e9f767SBen Widawsky &cxl_memdev_attribute_group, 116321e9f767SBen Widawsky &cxl_memdev_ram_attribute_group, 116421e9f767SBen Widawsky &cxl_memdev_pmem_attribute_group, 116521e9f767SBen Widawsky NULL, 116621e9f767SBen Widawsky }; 116721e9f767SBen Widawsky 116821e9f767SBen Widawsky static const struct device_type cxl_memdev_type = { 116921e9f767SBen Widawsky .name = "cxl_memdev", 117021e9f767SBen Widawsky .release = cxl_memdev_release, 117121e9f767SBen Widawsky .devnode = cxl_memdev_devnode, 117221e9f767SBen Widawsky .groups = cxl_memdev_attribute_groups, 117321e9f767SBen Widawsky }; 117421e9f767SBen Widawsky 117521e9f767SBen Widawsky static void cxl_memdev_shutdown(struct cxl_memdev *cxlmd) 117621e9f767SBen Widawsky { 117721e9f767SBen Widawsky down_write(&cxl_memdev_rwsem); 117821e9f767SBen Widawsky cxlmd->cxlm = NULL; 117921e9f767SBen Widawsky up_write(&cxl_memdev_rwsem); 118021e9f767SBen Widawsky } 118121e9f767SBen Widawsky 118221e9f767SBen Widawsky static void cxl_memdev_unregister(void *_cxlmd) 118321e9f767SBen Widawsky { 118421e9f767SBen Widawsky struct cxl_memdev *cxlmd = _cxlmd; 118521e9f767SBen Widawsky struct device *dev = &cxlmd->dev; 118621e9f767SBen Widawsky 118721e9f767SBen Widawsky cdev_device_del(&cxlmd->cdev, dev); 118821e9f767SBen Widawsky cxl_memdev_shutdown(cxlmd); 118921e9f767SBen Widawsky put_device(dev); 119021e9f767SBen Widawsky } 119121e9f767SBen Widawsky 119221e9f767SBen Widawsky static struct cxl_memdev *cxl_memdev_alloc(struct cxl_mem *cxlm) 119321e9f767SBen Widawsky { 119421e9f767SBen Widawsky struct pci_dev *pdev = cxlm->pdev; 119521e9f767SBen Widawsky struct cxl_memdev *cxlmd; 119621e9f767SBen Widawsky struct device *dev; 119721e9f767SBen Widawsky struct cdev *cdev; 119821e9f767SBen Widawsky int rc; 119921e9f767SBen Widawsky 120021e9f767SBen Widawsky cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL); 120121e9f767SBen Widawsky if (!cxlmd) 120221e9f767SBen Widawsky return ERR_PTR(-ENOMEM); 120321e9f767SBen Widawsky 120421e9f767SBen Widawsky rc = ida_alloc_range(&cxl_memdev_ida, 0, CXL_MEM_MAX_DEVS, GFP_KERNEL); 120521e9f767SBen Widawsky if (rc < 0) 120621e9f767SBen Widawsky goto err; 120721e9f767SBen Widawsky cxlmd->id = rc; 120821e9f767SBen Widawsky 120921e9f767SBen Widawsky dev = &cxlmd->dev; 121021e9f767SBen Widawsky device_initialize(dev); 121121e9f767SBen Widawsky dev->parent = &pdev->dev; 121221e9f767SBen Widawsky dev->bus = &cxl_bus_type; 121321e9f767SBen Widawsky dev->devt = MKDEV(cxl_mem_major, cxlmd->id); 121421e9f767SBen Widawsky dev->type = &cxl_memdev_type; 121521e9f767SBen Widawsky device_set_pm_not_required(dev); 121621e9f767SBen Widawsky 121721e9f767SBen Widawsky cdev = &cxlmd->cdev; 121821e9f767SBen Widawsky cdev_init(cdev, &cxl_memdev_fops); 121921e9f767SBen Widawsky return cxlmd; 122021e9f767SBen Widawsky 122121e9f767SBen Widawsky err: 122221e9f767SBen Widawsky kfree(cxlmd); 122321e9f767SBen Widawsky return ERR_PTR(rc); 122421e9f767SBen Widawsky } 122521e9f767SBen Widawsky 122621e9f767SBen Widawsky static int cxl_mem_add_memdev(struct cxl_mem *cxlm) 122721e9f767SBen Widawsky { 122821e9f767SBen Widawsky struct cxl_memdev *cxlmd; 122921e9f767SBen Widawsky struct device *dev; 123021e9f767SBen Widawsky struct cdev *cdev; 123121e9f767SBen Widawsky int rc; 123221e9f767SBen Widawsky 123321e9f767SBen Widawsky cxlmd = cxl_memdev_alloc(cxlm); 123421e9f767SBen Widawsky if (IS_ERR(cxlmd)) 123521e9f767SBen Widawsky return PTR_ERR(cxlmd); 123621e9f767SBen Widawsky 123721e9f767SBen Widawsky dev = &cxlmd->dev; 123821e9f767SBen Widawsky rc = dev_set_name(dev, "mem%d", cxlmd->id); 123921e9f767SBen Widawsky if (rc) 124021e9f767SBen Widawsky goto err; 124121e9f767SBen Widawsky 124221e9f767SBen Widawsky /* 124321e9f767SBen Widawsky * Activate ioctl operations, no cxl_memdev_rwsem manipulation 124421e9f767SBen Widawsky * needed as this is ordered with cdev_add() publishing the device. 124521e9f767SBen Widawsky */ 124621e9f767SBen Widawsky cxlmd->cxlm = cxlm; 124721e9f767SBen Widawsky 124821e9f767SBen Widawsky cdev = &cxlmd->cdev; 124921e9f767SBen Widawsky rc = cdev_device_add(cdev, dev); 125021e9f767SBen Widawsky if (rc) 125121e9f767SBen Widawsky goto err; 125221e9f767SBen Widawsky 125321e9f767SBen Widawsky return devm_add_action_or_reset(dev->parent, cxl_memdev_unregister, 125421e9f767SBen Widawsky cxlmd); 125521e9f767SBen Widawsky 125621e9f767SBen Widawsky err: 125721e9f767SBen Widawsky /* 125821e9f767SBen Widawsky * The cdev was briefly live, shutdown any ioctl operations that 125921e9f767SBen Widawsky * saw that state. 126021e9f767SBen Widawsky */ 126121e9f767SBen Widawsky cxl_memdev_shutdown(cxlmd); 126221e9f767SBen Widawsky put_device(dev); 126321e9f767SBen Widawsky return rc; 126421e9f767SBen Widawsky } 126521e9f767SBen Widawsky 126621e9f767SBen Widawsky static int cxl_xfer_log(struct cxl_mem *cxlm, uuid_t *uuid, u32 size, u8 *out) 126721e9f767SBen Widawsky { 126821e9f767SBen Widawsky u32 remaining = size; 126921e9f767SBen Widawsky u32 offset = 0; 127021e9f767SBen Widawsky 127121e9f767SBen Widawsky while (remaining) { 127221e9f767SBen Widawsky u32 xfer_size = min_t(u32, remaining, cxlm->payload_size); 127321e9f767SBen Widawsky struct cxl_mbox_get_log { 127421e9f767SBen Widawsky uuid_t uuid; 127521e9f767SBen Widawsky __le32 offset; 127621e9f767SBen Widawsky __le32 length; 127721e9f767SBen Widawsky } __packed log = { 127821e9f767SBen Widawsky .uuid = *uuid, 127921e9f767SBen Widawsky .offset = cpu_to_le32(offset), 128021e9f767SBen Widawsky .length = cpu_to_le32(xfer_size) 128121e9f767SBen Widawsky }; 128221e9f767SBen Widawsky int rc; 128321e9f767SBen Widawsky 128421e9f767SBen Widawsky rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_LOG, &log, 128521e9f767SBen Widawsky sizeof(log), out, xfer_size); 128621e9f767SBen Widawsky if (rc < 0) 128721e9f767SBen Widawsky return rc; 128821e9f767SBen Widawsky 128921e9f767SBen Widawsky out += xfer_size; 129021e9f767SBen Widawsky remaining -= xfer_size; 129121e9f767SBen Widawsky offset += xfer_size; 129221e9f767SBen Widawsky } 129321e9f767SBen Widawsky 129421e9f767SBen Widawsky return 0; 129521e9f767SBen Widawsky } 129621e9f767SBen Widawsky 129721e9f767SBen Widawsky /** 129821e9f767SBen Widawsky * cxl_walk_cel() - Walk through the Command Effects Log. 129921e9f767SBen Widawsky * @cxlm: Device. 130021e9f767SBen Widawsky * @size: Length of the Command Effects Log. 130121e9f767SBen Widawsky * @cel: CEL 130221e9f767SBen Widawsky * 130321e9f767SBen Widawsky * Iterate over each entry in the CEL and determine if the driver supports the 130421e9f767SBen Widawsky * command. If so, the command is enabled for the device and can be used later. 130521e9f767SBen Widawsky */ 130621e9f767SBen Widawsky static void cxl_walk_cel(struct cxl_mem *cxlm, size_t size, u8 *cel) 130721e9f767SBen Widawsky { 130821e9f767SBen Widawsky struct cel_entry { 130921e9f767SBen Widawsky __le16 opcode; 131021e9f767SBen Widawsky __le16 effect; 131121e9f767SBen Widawsky } __packed * cel_entry; 131221e9f767SBen Widawsky const int cel_entries = size / sizeof(*cel_entry); 131321e9f767SBen Widawsky int i; 131421e9f767SBen Widawsky 131521e9f767SBen Widawsky cel_entry = (struct cel_entry *)cel; 131621e9f767SBen Widawsky 131721e9f767SBen Widawsky for (i = 0; i < cel_entries; i++) { 131821e9f767SBen Widawsky u16 opcode = le16_to_cpu(cel_entry[i].opcode); 131921e9f767SBen Widawsky struct cxl_mem_command *cmd = cxl_mem_find_command(opcode); 132021e9f767SBen Widawsky 132121e9f767SBen Widawsky if (!cmd) { 132221e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, 132321e9f767SBen Widawsky "Opcode 0x%04x unsupported by driver", opcode); 132421e9f767SBen Widawsky continue; 132521e9f767SBen Widawsky } 132621e9f767SBen Widawsky 132721e9f767SBen Widawsky set_bit(cmd->info.id, cxlm->enabled_cmds); 132821e9f767SBen Widawsky } 132921e9f767SBen Widawsky } 133021e9f767SBen Widawsky 133121e9f767SBen Widawsky struct cxl_mbox_get_supported_logs { 133221e9f767SBen Widawsky __le16 entries; 133321e9f767SBen Widawsky u8 rsvd[6]; 133421e9f767SBen Widawsky struct gsl_entry { 133521e9f767SBen Widawsky uuid_t uuid; 133621e9f767SBen Widawsky __le32 size; 133721e9f767SBen Widawsky } __packed entry[]; 133821e9f767SBen Widawsky } __packed; 133921e9f767SBen Widawsky 134021e9f767SBen Widawsky static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_mem *cxlm) 134121e9f767SBen Widawsky { 134221e9f767SBen Widawsky struct cxl_mbox_get_supported_logs *ret; 134321e9f767SBen Widawsky int rc; 134421e9f767SBen Widawsky 134521e9f767SBen Widawsky ret = kvmalloc(cxlm->payload_size, GFP_KERNEL); 134621e9f767SBen Widawsky if (!ret) 134721e9f767SBen Widawsky return ERR_PTR(-ENOMEM); 134821e9f767SBen Widawsky 134921e9f767SBen Widawsky rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_SUPPORTED_LOGS, NULL, 135021e9f767SBen Widawsky 0, ret, cxlm->payload_size); 135121e9f767SBen Widawsky if (rc < 0) { 135221e9f767SBen Widawsky kvfree(ret); 135321e9f767SBen Widawsky return ERR_PTR(rc); 135421e9f767SBen Widawsky } 135521e9f767SBen Widawsky 135621e9f767SBen Widawsky return ret; 135721e9f767SBen Widawsky } 135821e9f767SBen Widawsky 135921e9f767SBen Widawsky /** 136021e9f767SBen Widawsky * cxl_mem_enumerate_cmds() - Enumerate commands for a device. 136121e9f767SBen Widawsky * @cxlm: The device. 136221e9f767SBen Widawsky * 136321e9f767SBen Widawsky * Returns 0 if enumerate completed successfully. 136421e9f767SBen Widawsky * 136521e9f767SBen Widawsky * CXL devices have optional support for certain commands. This function will 136621e9f767SBen Widawsky * determine the set of supported commands for the hardware and update the 136721e9f767SBen Widawsky * enabled_cmds bitmap in the @cxlm. 136821e9f767SBen Widawsky */ 136921e9f767SBen Widawsky static int cxl_mem_enumerate_cmds(struct cxl_mem *cxlm) 137021e9f767SBen Widawsky { 137121e9f767SBen Widawsky struct cxl_mbox_get_supported_logs *gsl; 137221e9f767SBen Widawsky struct device *dev = &cxlm->pdev->dev; 137321e9f767SBen Widawsky struct cxl_mem_command *cmd; 137421e9f767SBen Widawsky int i, rc; 137521e9f767SBen Widawsky 137621e9f767SBen Widawsky gsl = cxl_get_gsl(cxlm); 137721e9f767SBen Widawsky if (IS_ERR(gsl)) 137821e9f767SBen Widawsky return PTR_ERR(gsl); 137921e9f767SBen Widawsky 138021e9f767SBen Widawsky rc = -ENOENT; 138121e9f767SBen Widawsky for (i = 0; i < le16_to_cpu(gsl->entries); i++) { 138221e9f767SBen Widawsky u32 size = le32_to_cpu(gsl->entry[i].size); 138321e9f767SBen Widawsky uuid_t uuid = gsl->entry[i].uuid; 138421e9f767SBen Widawsky u8 *log; 138521e9f767SBen Widawsky 138621e9f767SBen Widawsky dev_dbg(dev, "Found LOG type %pU of size %d", &uuid, size); 138721e9f767SBen Widawsky 138821e9f767SBen Widawsky if (!uuid_equal(&uuid, &log_uuid[CEL_UUID])) 138921e9f767SBen Widawsky continue; 139021e9f767SBen Widawsky 139121e9f767SBen Widawsky log = kvmalloc(size, GFP_KERNEL); 139221e9f767SBen Widawsky if (!log) { 139321e9f767SBen Widawsky rc = -ENOMEM; 139421e9f767SBen Widawsky goto out; 139521e9f767SBen Widawsky } 139621e9f767SBen Widawsky 139721e9f767SBen Widawsky rc = cxl_xfer_log(cxlm, &uuid, size, log); 139821e9f767SBen Widawsky if (rc) { 139921e9f767SBen Widawsky kvfree(log); 140021e9f767SBen Widawsky goto out; 140121e9f767SBen Widawsky } 140221e9f767SBen Widawsky 140321e9f767SBen Widawsky cxl_walk_cel(cxlm, size, log); 140421e9f767SBen Widawsky kvfree(log); 140521e9f767SBen Widawsky 140621e9f767SBen Widawsky /* In case CEL was bogus, enable some default commands. */ 140721e9f767SBen Widawsky cxl_for_each_cmd(cmd) 140821e9f767SBen Widawsky if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE) 140921e9f767SBen Widawsky set_bit(cmd->info.id, cxlm->enabled_cmds); 141021e9f767SBen Widawsky 141121e9f767SBen Widawsky /* Found the required CEL */ 141221e9f767SBen Widawsky rc = 0; 141321e9f767SBen Widawsky } 141421e9f767SBen Widawsky 141521e9f767SBen Widawsky out: 141621e9f767SBen Widawsky kvfree(gsl); 141721e9f767SBen Widawsky return rc; 141821e9f767SBen Widawsky } 141921e9f767SBen Widawsky 142021e9f767SBen Widawsky /** 142121e9f767SBen Widawsky * cxl_mem_identify() - Send the IDENTIFY command to the device. 142221e9f767SBen Widawsky * @cxlm: The device to identify. 142321e9f767SBen Widawsky * 142421e9f767SBen Widawsky * Return: 0 if identify was executed successfully. 142521e9f767SBen Widawsky * 142621e9f767SBen Widawsky * This will dispatch the identify command to the device and on success populate 142721e9f767SBen Widawsky * structures to be exported to sysfs. 142821e9f767SBen Widawsky */ 142921e9f767SBen Widawsky static int cxl_mem_identify(struct cxl_mem *cxlm) 143021e9f767SBen Widawsky { 143121e9f767SBen Widawsky /* See CXL 2.0 Table 175 Identify Memory Device Output Payload */ 143221e9f767SBen Widawsky struct cxl_mbox_identify { 143321e9f767SBen Widawsky char fw_revision[0x10]; 143421e9f767SBen Widawsky __le64 total_capacity; 143521e9f767SBen Widawsky __le64 volatile_capacity; 143621e9f767SBen Widawsky __le64 persistent_capacity; 143721e9f767SBen Widawsky __le64 partition_align; 143821e9f767SBen Widawsky __le16 info_event_log_size; 143921e9f767SBen Widawsky __le16 warning_event_log_size; 144021e9f767SBen Widawsky __le16 failure_event_log_size; 144121e9f767SBen Widawsky __le16 fatal_event_log_size; 144221e9f767SBen Widawsky __le32 lsa_size; 144321e9f767SBen Widawsky u8 poison_list_max_mer[3]; 144421e9f767SBen Widawsky __le16 inject_poison_limit; 144521e9f767SBen Widawsky u8 poison_caps; 144621e9f767SBen Widawsky u8 qos_telemetry_caps; 144721e9f767SBen Widawsky } __packed id; 144821e9f767SBen Widawsky int rc; 144921e9f767SBen Widawsky 145021e9f767SBen Widawsky rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_IDENTIFY, NULL, 0, &id, 145121e9f767SBen Widawsky sizeof(id)); 145221e9f767SBen Widawsky if (rc < 0) 145321e9f767SBen Widawsky return rc; 145421e9f767SBen Widawsky 145521e9f767SBen Widawsky /* 145621e9f767SBen Widawsky * TODO: enumerate DPA map, as 'ram' and 'pmem' do not alias. 145721e9f767SBen Widawsky * For now, only the capacity is exported in sysfs 145821e9f767SBen Widawsky */ 145921e9f767SBen Widawsky cxlm->ram_range.start = 0; 146021e9f767SBen Widawsky cxlm->ram_range.end = le64_to_cpu(id.volatile_capacity) * SZ_256M - 1; 146121e9f767SBen Widawsky 146221e9f767SBen Widawsky cxlm->pmem_range.start = 0; 146321e9f767SBen Widawsky cxlm->pmem_range.end = 146421e9f767SBen Widawsky le64_to_cpu(id.persistent_capacity) * SZ_256M - 1; 146521e9f767SBen Widawsky 1466199cf8c3SVishal Verma cxlm->lsa_size = le32_to_cpu(id.lsa_size); 146721e9f767SBen Widawsky memcpy(cxlm->firmware_version, id.fw_revision, sizeof(id.fw_revision)); 146821e9f767SBen Widawsky 146921e9f767SBen Widawsky return 0; 147021e9f767SBen Widawsky } 147121e9f767SBen Widawsky 147221e9f767SBen Widawsky static int cxl_mem_probe(struct pci_dev *pdev, const struct pci_device_id *id) 147321e9f767SBen Widawsky { 14741b0a1a2aSBen Widawsky struct cxl_mem *cxlm; 14751d5a4159SBen Widawsky int rc; 147621e9f767SBen Widawsky 147721e9f767SBen Widawsky rc = pcim_enable_device(pdev); 147821e9f767SBen Widawsky if (rc) 147921e9f767SBen Widawsky return rc; 148021e9f767SBen Widawsky 14811b0a1a2aSBen Widawsky cxlm = cxl_mem_create(pdev); 14821b0a1a2aSBen Widawsky if (IS_ERR(cxlm)) 14831b0a1a2aSBen Widawsky return PTR_ERR(cxlm); 14841b0a1a2aSBen Widawsky 148521e9f767SBen Widawsky rc = cxl_mem_setup_regs(cxlm); 148621e9f767SBen Widawsky if (rc) 148721e9f767SBen Widawsky return rc; 148821e9f767SBen Widawsky 148921e9f767SBen Widawsky rc = cxl_mem_setup_mailbox(cxlm); 149021e9f767SBen Widawsky if (rc) 149121e9f767SBen Widawsky return rc; 149221e9f767SBen Widawsky 149321e9f767SBen Widawsky rc = cxl_mem_enumerate_cmds(cxlm); 149421e9f767SBen Widawsky if (rc) 149521e9f767SBen Widawsky return rc; 149621e9f767SBen Widawsky 149721e9f767SBen Widawsky rc = cxl_mem_identify(cxlm); 149821e9f767SBen Widawsky if (rc) 149921e9f767SBen Widawsky return rc; 150021e9f767SBen Widawsky 150121e9f767SBen Widawsky return cxl_mem_add_memdev(cxlm); 150221e9f767SBen Widawsky } 150321e9f767SBen Widawsky 150421e9f767SBen Widawsky static const struct pci_device_id cxl_mem_pci_tbl[] = { 150521e9f767SBen Widawsky /* PCI class code for CXL.mem Type-3 Devices */ 150621e9f767SBen Widawsky { PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)}, 150721e9f767SBen Widawsky { /* terminate list */ }, 150821e9f767SBen Widawsky }; 150921e9f767SBen Widawsky MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl); 151021e9f767SBen Widawsky 151121e9f767SBen Widawsky static struct pci_driver cxl_mem_driver = { 151221e9f767SBen Widawsky .name = KBUILD_MODNAME, 151321e9f767SBen Widawsky .id_table = cxl_mem_pci_tbl, 151421e9f767SBen Widawsky .probe = cxl_mem_probe, 151521e9f767SBen Widawsky .driver = { 151621e9f767SBen Widawsky .probe_type = PROBE_PREFER_ASYNCHRONOUS, 151721e9f767SBen Widawsky }, 151821e9f767SBen Widawsky }; 151921e9f767SBen Widawsky 152021e9f767SBen Widawsky static __init int cxl_mem_init(void) 152121e9f767SBen Widawsky { 152221e9f767SBen Widawsky struct dentry *mbox_debugfs; 152321e9f767SBen Widawsky dev_t devt; 152421e9f767SBen Widawsky int rc; 152521e9f767SBen Widawsky 152621e9f767SBen Widawsky /* Double check the anonymous union trickery in struct cxl_regs */ 152721e9f767SBen Widawsky BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) != 152821e9f767SBen Widawsky offsetof(struct cxl_regs, device_regs.memdev)); 152921e9f767SBen Widawsky 153021e9f767SBen Widawsky rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); 153121e9f767SBen Widawsky if (rc) 153221e9f767SBen Widawsky return rc; 153321e9f767SBen Widawsky 153421e9f767SBen Widawsky cxl_mem_major = MAJOR(devt); 153521e9f767SBen Widawsky 153621e9f767SBen Widawsky rc = pci_register_driver(&cxl_mem_driver); 153721e9f767SBen Widawsky if (rc) { 153821e9f767SBen Widawsky unregister_chrdev_region(MKDEV(cxl_mem_major, 0), 153921e9f767SBen Widawsky CXL_MEM_MAX_DEVS); 154021e9f767SBen Widawsky return rc; 154121e9f767SBen Widawsky } 154221e9f767SBen Widawsky 154321e9f767SBen Widawsky cxl_debugfs = debugfs_create_dir("cxl", NULL); 154421e9f767SBen Widawsky mbox_debugfs = debugfs_create_dir("mbox", cxl_debugfs); 154521e9f767SBen Widawsky debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs, 154621e9f767SBen Widawsky &cxl_raw_allow_all); 154721e9f767SBen Widawsky 154821e9f767SBen Widawsky return 0; 154921e9f767SBen Widawsky } 155021e9f767SBen Widawsky 155121e9f767SBen Widawsky static __exit void cxl_mem_exit(void) 155221e9f767SBen Widawsky { 155321e9f767SBen Widawsky debugfs_remove_recursive(cxl_debugfs); 155421e9f767SBen Widawsky pci_unregister_driver(&cxl_mem_driver); 155521e9f767SBen Widawsky unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS); 155621e9f767SBen Widawsky } 155721e9f767SBen Widawsky 155821e9f767SBen Widawsky MODULE_LICENSE("GPL v2"); 155921e9f767SBen Widawsky module_init(cxl_mem_init); 156021e9f767SBen Widawsky module_exit(cxl_mem_exit); 156121e9f767SBen Widawsky MODULE_IMPORT_NS(CXL); 1562