121e9f767SBen Widawsky // SPDX-License-Identifier: GPL-2.0-only 221e9f767SBen Widawsky /* Copyright(c) 2020 Intel Corporation. All rights reserved. */ 321e9f767SBen Widawsky #include <uapi/linux/cxl_mem.h> 421e9f767SBen Widawsky #include <linux/security.h> 521e9f767SBen Widawsky #include <linux/debugfs.h> 621e9f767SBen Widawsky #include <linux/module.h> 721e9f767SBen Widawsky #include <linux/sizes.h> 821e9f767SBen Widawsky #include <linux/mutex.h> 930af9729SIra Weiny #include <linux/list.h> 1021e9f767SBen Widawsky #include <linux/cdev.h> 1121e9f767SBen Widawsky #include <linux/idr.h> 1221e9f767SBen Widawsky #include <linux/pci.h> 1321e9f767SBen Widawsky #include <linux/io.h> 1421e9f767SBen Widawsky #include <linux/io-64-nonatomic-lo-hi.h> 1521e9f767SBen Widawsky #include "pci.h" 1621e9f767SBen Widawsky #include "cxl.h" 1721e9f767SBen Widawsky #include "mem.h" 1821e9f767SBen Widawsky 1921e9f767SBen Widawsky /** 2021e9f767SBen Widawsky * DOC: cxl pci 2121e9f767SBen Widawsky * 2221e9f767SBen Widawsky * This implements the PCI exclusive functionality for a CXL device as it is 2321e9f767SBen Widawsky * defined by the Compute Express Link specification. CXL devices may surface 2421e9f767SBen Widawsky * certain functionality even if it isn't CXL enabled. 2521e9f767SBen Widawsky * 2621e9f767SBen Widawsky * The driver has several responsibilities, mainly: 2721e9f767SBen Widawsky * - Create the memX device and register on the CXL bus. 2821e9f767SBen Widawsky * - Enumerate device's register interface and map them. 2921e9f767SBen Widawsky * - Probe the device attributes to establish sysfs interface. 3021e9f767SBen Widawsky * - Provide an IOCTL interface to userspace to communicate with the device for 3121e9f767SBen Widawsky * things like firmware update. 3221e9f767SBen Widawsky */ 3321e9f767SBen Widawsky 3421e9f767SBen Widawsky #define cxl_doorbell_busy(cxlm) \ 3521e9f767SBen Widawsky (readl((cxlm)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \ 3621e9f767SBen Widawsky CXLDEV_MBOX_CTRL_DOORBELL) 3721e9f767SBen Widawsky 3821e9f767SBen Widawsky /* CXL 2.0 - 8.2.8.4 */ 3921e9f767SBen Widawsky #define CXL_MAILBOX_TIMEOUT_MS (2 * HZ) 4021e9f767SBen Widawsky 4121e9f767SBen Widawsky enum opcode { 4221e9f767SBen Widawsky CXL_MBOX_OP_INVALID = 0x0000, 4321e9f767SBen Widawsky CXL_MBOX_OP_RAW = CXL_MBOX_OP_INVALID, 4421e9f767SBen Widawsky CXL_MBOX_OP_GET_FW_INFO = 0x0200, 4521e9f767SBen Widawsky CXL_MBOX_OP_ACTIVATE_FW = 0x0202, 4621e9f767SBen Widawsky CXL_MBOX_OP_GET_SUPPORTED_LOGS = 0x0400, 4721e9f767SBen Widawsky CXL_MBOX_OP_GET_LOG = 0x0401, 4821e9f767SBen Widawsky CXL_MBOX_OP_IDENTIFY = 0x4000, 4921e9f767SBen Widawsky CXL_MBOX_OP_GET_PARTITION_INFO = 0x4100, 5021e9f767SBen Widawsky CXL_MBOX_OP_SET_PARTITION_INFO = 0x4101, 5121e9f767SBen Widawsky CXL_MBOX_OP_GET_LSA = 0x4102, 5221e9f767SBen Widawsky CXL_MBOX_OP_SET_LSA = 0x4103, 5321e9f767SBen Widawsky CXL_MBOX_OP_GET_HEALTH_INFO = 0x4200, 5421e9f767SBen Widawsky CXL_MBOX_OP_SET_SHUTDOWN_STATE = 0x4204, 5521e9f767SBen Widawsky CXL_MBOX_OP_SCAN_MEDIA = 0x4304, 5621e9f767SBen Widawsky CXL_MBOX_OP_GET_SCAN_MEDIA = 0x4305, 5721e9f767SBen Widawsky CXL_MBOX_OP_MAX = 0x10000 5821e9f767SBen Widawsky }; 5921e9f767SBen Widawsky 6021e9f767SBen Widawsky /** 6121e9f767SBen Widawsky * struct mbox_cmd - A command to be submitted to hardware. 6221e9f767SBen Widawsky * @opcode: (input) The command set and command submitted to hardware. 6321e9f767SBen Widawsky * @payload_in: (input) Pointer to the input payload. 6421e9f767SBen Widawsky * @payload_out: (output) Pointer to the output payload. Must be allocated by 6521e9f767SBen Widawsky * the caller. 6621e9f767SBen Widawsky * @size_in: (input) Number of bytes to load from @payload_in. 6721e9f767SBen Widawsky * @size_out: (input) Max number of bytes loaded into @payload_out. 6821e9f767SBen Widawsky * (output) Number of bytes generated by the device. For fixed size 6921e9f767SBen Widawsky * outputs commands this is always expected to be deterministic. For 7021e9f767SBen Widawsky * variable sized output commands, it tells the exact number of bytes 7121e9f767SBen Widawsky * written. 7221e9f767SBen Widawsky * @return_code: (output) Error code returned from hardware. 7321e9f767SBen Widawsky * 7421e9f767SBen Widawsky * This is the primary mechanism used to send commands to the hardware. 7521e9f767SBen Widawsky * All the fields except @payload_* correspond exactly to the fields described in 7621e9f767SBen Widawsky * Command Register section of the CXL 2.0 8.2.8.4.5. @payload_in and 7721e9f767SBen Widawsky * @payload_out are written to, and read from the Command Payload Registers 7821e9f767SBen Widawsky * defined in CXL 2.0 8.2.8.4.8. 7921e9f767SBen Widawsky */ 8021e9f767SBen Widawsky struct mbox_cmd { 8121e9f767SBen Widawsky u16 opcode; 8221e9f767SBen Widawsky void *payload_in; 8321e9f767SBen Widawsky void *payload_out; 8421e9f767SBen Widawsky size_t size_in; 8521e9f767SBen Widawsky size_t size_out; 8621e9f767SBen Widawsky u16 return_code; 8721e9f767SBen Widawsky #define CXL_MBOX_SUCCESS 0 8821e9f767SBen Widawsky }; 8921e9f767SBen Widawsky 9021e9f767SBen Widawsky static int cxl_mem_major; 9121e9f767SBen Widawsky static DEFINE_IDA(cxl_memdev_ida); 9221e9f767SBen Widawsky static DECLARE_RWSEM(cxl_memdev_rwsem); 9321e9f767SBen Widawsky static struct dentry *cxl_debugfs; 9421e9f767SBen Widawsky static bool cxl_raw_allow_all; 9521e9f767SBen Widawsky 9621e9f767SBen Widawsky enum { 9721e9f767SBen Widawsky CEL_UUID, 9821e9f767SBen Widawsky VENDOR_DEBUG_UUID, 9921e9f767SBen Widawsky }; 10021e9f767SBen Widawsky 10121e9f767SBen Widawsky /* See CXL 2.0 Table 170. Get Log Input Payload */ 10221e9f767SBen Widawsky static const uuid_t log_uuid[] = { 10321e9f767SBen Widawsky [CEL_UUID] = UUID_INIT(0xda9c0b5, 0xbf41, 0x4b78, 0x8f, 0x79, 0x96, 10421e9f767SBen Widawsky 0xb1, 0x62, 0x3b, 0x3f, 0x17), 10521e9f767SBen Widawsky [VENDOR_DEBUG_UUID] = UUID_INIT(0xe1819d9, 0x11a9, 0x400c, 0x81, 0x1f, 10621e9f767SBen Widawsky 0xd6, 0x07, 0x19, 0x40, 0x3d, 0x86), 10721e9f767SBen Widawsky }; 10821e9f767SBen Widawsky 10921e9f767SBen Widawsky /** 11021e9f767SBen Widawsky * struct cxl_mem_command - Driver representation of a memory device command 11121e9f767SBen Widawsky * @info: Command information as it exists for the UAPI 11221e9f767SBen Widawsky * @opcode: The actual bits used for the mailbox protocol 11321e9f767SBen Widawsky * @flags: Set of flags effecting driver behavior. 11421e9f767SBen Widawsky * 11521e9f767SBen Widawsky * * %CXL_CMD_FLAG_FORCE_ENABLE: In cases of error, commands with this flag 11621e9f767SBen Widawsky * will be enabled by the driver regardless of what hardware may have 11721e9f767SBen Widawsky * advertised. 11821e9f767SBen Widawsky * 11921e9f767SBen Widawsky * The cxl_mem_command is the driver's internal representation of commands that 12021e9f767SBen Widawsky * are supported by the driver. Some of these commands may not be supported by 12121e9f767SBen Widawsky * the hardware. The driver will use @info to validate the fields passed in by 12221e9f767SBen Widawsky * the user then submit the @opcode to the hardware. 12321e9f767SBen Widawsky * 12421e9f767SBen Widawsky * See struct cxl_command_info. 12521e9f767SBen Widawsky */ 12621e9f767SBen Widawsky struct cxl_mem_command { 12721e9f767SBen Widawsky struct cxl_command_info info; 12821e9f767SBen Widawsky enum opcode opcode; 12921e9f767SBen Widawsky u32 flags; 13021e9f767SBen Widawsky #define CXL_CMD_FLAG_NONE 0 13121e9f767SBen Widawsky #define CXL_CMD_FLAG_FORCE_ENABLE BIT(0) 13221e9f767SBen Widawsky }; 13321e9f767SBen Widawsky 13421e9f767SBen Widawsky #define CXL_CMD(_id, sin, sout, _flags) \ 13521e9f767SBen Widawsky [CXL_MEM_COMMAND_ID_##_id] = { \ 13621e9f767SBen Widawsky .info = { \ 13721e9f767SBen Widawsky .id = CXL_MEM_COMMAND_ID_##_id, \ 13821e9f767SBen Widawsky .size_in = sin, \ 13921e9f767SBen Widawsky .size_out = sout, \ 14021e9f767SBen Widawsky }, \ 14121e9f767SBen Widawsky .opcode = CXL_MBOX_OP_##_id, \ 14221e9f767SBen Widawsky .flags = _flags, \ 14321e9f767SBen Widawsky } 14421e9f767SBen Widawsky 14521e9f767SBen Widawsky /* 14621e9f767SBen Widawsky * This table defines the supported mailbox commands for the driver. This table 14721e9f767SBen Widawsky * is made up of a UAPI structure. Non-negative values as parameters in the 14821e9f767SBen Widawsky * table will be validated against the user's input. For example, if size_in is 14921e9f767SBen Widawsky * 0, and the user passed in 1, it is an error. 15021e9f767SBen Widawsky */ 15121e9f767SBen Widawsky static struct cxl_mem_command mem_commands[CXL_MEM_COMMAND_ID_MAX] = { 15221e9f767SBen Widawsky CXL_CMD(IDENTIFY, 0, 0x43, CXL_CMD_FLAG_FORCE_ENABLE), 15321e9f767SBen Widawsky #ifdef CONFIG_CXL_MEM_RAW_COMMANDS 15421e9f767SBen Widawsky CXL_CMD(RAW, ~0, ~0, 0), 15521e9f767SBen Widawsky #endif 15621e9f767SBen Widawsky CXL_CMD(GET_SUPPORTED_LOGS, 0, ~0, CXL_CMD_FLAG_FORCE_ENABLE), 15721e9f767SBen Widawsky CXL_CMD(GET_FW_INFO, 0, 0x50, 0), 15821e9f767SBen Widawsky CXL_CMD(GET_PARTITION_INFO, 0, 0x20, 0), 15921e9f767SBen Widawsky CXL_CMD(GET_LSA, 0x8, ~0, 0), 16021e9f767SBen Widawsky CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0), 16121e9f767SBen Widawsky CXL_CMD(GET_LOG, 0x18, ~0, CXL_CMD_FLAG_FORCE_ENABLE), 16221e9f767SBen Widawsky }; 16321e9f767SBen Widawsky 16421e9f767SBen Widawsky /* 16521e9f767SBen Widawsky * Commands that RAW doesn't permit. The rationale for each: 16621e9f767SBen Widawsky * 16721e9f767SBen Widawsky * CXL_MBOX_OP_ACTIVATE_FW: Firmware activation requires adjustment / 16821e9f767SBen Widawsky * coordination of transaction timeout values at the root bridge level. 16921e9f767SBen Widawsky * 17021e9f767SBen Widawsky * CXL_MBOX_OP_SET_PARTITION_INFO: The device memory map may change live 17121e9f767SBen Widawsky * and needs to be coordinated with HDM updates. 17221e9f767SBen Widawsky * 17321e9f767SBen Widawsky * CXL_MBOX_OP_SET_LSA: The label storage area may be cached by the 17421e9f767SBen Widawsky * driver and any writes from userspace invalidates those contents. 17521e9f767SBen Widawsky * 17621e9f767SBen Widawsky * CXL_MBOX_OP_SET_SHUTDOWN_STATE: Set shutdown state assumes no writes 17721e9f767SBen Widawsky * to the device after it is marked clean, userspace can not make that 17821e9f767SBen Widawsky * assertion. 17921e9f767SBen Widawsky * 18021e9f767SBen Widawsky * CXL_MBOX_OP_[GET_]SCAN_MEDIA: The kernel provides a native error list that 18121e9f767SBen Widawsky * is kept up to date with patrol notifications and error management. 18221e9f767SBen Widawsky */ 18321e9f767SBen Widawsky static u16 cxl_disabled_raw_commands[] = { 18421e9f767SBen Widawsky CXL_MBOX_OP_ACTIVATE_FW, 18521e9f767SBen Widawsky CXL_MBOX_OP_SET_PARTITION_INFO, 18621e9f767SBen Widawsky CXL_MBOX_OP_SET_LSA, 18721e9f767SBen Widawsky CXL_MBOX_OP_SET_SHUTDOWN_STATE, 18821e9f767SBen Widawsky CXL_MBOX_OP_SCAN_MEDIA, 18921e9f767SBen Widawsky CXL_MBOX_OP_GET_SCAN_MEDIA, 19021e9f767SBen Widawsky }; 19121e9f767SBen Widawsky 19221e9f767SBen Widawsky /* 19321e9f767SBen Widawsky * Command sets that RAW doesn't permit. All opcodes in this set are 19421e9f767SBen Widawsky * disabled because they pass plain text security payloads over the 19521e9f767SBen Widawsky * user/kernel boundary. This functionality is intended to be wrapped 19621e9f767SBen Widawsky * behind the keys ABI which allows for encrypted payloads in the UAPI 19721e9f767SBen Widawsky */ 19821e9f767SBen Widawsky static u8 security_command_sets[] = { 19921e9f767SBen Widawsky 0x44, /* Sanitize */ 20021e9f767SBen Widawsky 0x45, /* Persistent Memory Data-at-rest Security */ 20121e9f767SBen Widawsky 0x46, /* Security Passthrough */ 20221e9f767SBen Widawsky }; 20321e9f767SBen Widawsky 20421e9f767SBen Widawsky #define cxl_for_each_cmd(cmd) \ 20521e9f767SBen Widawsky for ((cmd) = &mem_commands[0]; \ 20621e9f767SBen Widawsky ((cmd) - mem_commands) < ARRAY_SIZE(mem_commands); (cmd)++) 20721e9f767SBen Widawsky 20821e9f767SBen Widawsky #define cxl_cmd_count ARRAY_SIZE(mem_commands) 20921e9f767SBen Widawsky 21021e9f767SBen Widawsky static int cxl_mem_wait_for_doorbell(struct cxl_mem *cxlm) 21121e9f767SBen Widawsky { 21221e9f767SBen Widawsky const unsigned long start = jiffies; 21321e9f767SBen Widawsky unsigned long end = start; 21421e9f767SBen Widawsky 21521e9f767SBen Widawsky while (cxl_doorbell_busy(cxlm)) { 21621e9f767SBen Widawsky end = jiffies; 21721e9f767SBen Widawsky 21821e9f767SBen Widawsky if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) { 21921e9f767SBen Widawsky /* Check again in case preempted before timeout test */ 22021e9f767SBen Widawsky if (!cxl_doorbell_busy(cxlm)) 22121e9f767SBen Widawsky break; 22221e9f767SBen Widawsky return -ETIMEDOUT; 22321e9f767SBen Widawsky } 22421e9f767SBen Widawsky cpu_relax(); 22521e9f767SBen Widawsky } 22621e9f767SBen Widawsky 22721e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, "Doorbell wait took %dms", 22821e9f767SBen Widawsky jiffies_to_msecs(end) - jiffies_to_msecs(start)); 22921e9f767SBen Widawsky return 0; 23021e9f767SBen Widawsky } 23121e9f767SBen Widawsky 23221e9f767SBen Widawsky static bool cxl_is_security_command(u16 opcode) 23321e9f767SBen Widawsky { 23421e9f767SBen Widawsky int i; 23521e9f767SBen Widawsky 23621e9f767SBen Widawsky for (i = 0; i < ARRAY_SIZE(security_command_sets); i++) 23721e9f767SBen Widawsky if (security_command_sets[i] == (opcode >> 8)) 23821e9f767SBen Widawsky return true; 23921e9f767SBen Widawsky return false; 24021e9f767SBen Widawsky } 24121e9f767SBen Widawsky 24221e9f767SBen Widawsky static void cxl_mem_mbox_timeout(struct cxl_mem *cxlm, 24321e9f767SBen Widawsky struct mbox_cmd *mbox_cmd) 24421e9f767SBen Widawsky { 24521e9f767SBen Widawsky struct device *dev = &cxlm->pdev->dev; 24621e9f767SBen Widawsky 24721e9f767SBen Widawsky dev_dbg(dev, "Mailbox command (opcode: %#x size: %zub) timed out\n", 24821e9f767SBen Widawsky mbox_cmd->opcode, mbox_cmd->size_in); 24921e9f767SBen Widawsky } 25021e9f767SBen Widawsky 25121e9f767SBen Widawsky /** 25221e9f767SBen Widawsky * __cxl_mem_mbox_send_cmd() - Execute a mailbox command 25321e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 25421e9f767SBen Widawsky * @mbox_cmd: Command to send to the memory device. 25521e9f767SBen Widawsky * 25621e9f767SBen Widawsky * Context: Any context. Expects mbox_mutex to be held. 25721e9f767SBen Widawsky * Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success. 25821e9f767SBen Widawsky * Caller should check the return code in @mbox_cmd to make sure it 25921e9f767SBen Widawsky * succeeded. 26021e9f767SBen Widawsky * 26121e9f767SBen Widawsky * This is a generic form of the CXL mailbox send command thus only using the 26221e9f767SBen Widawsky * registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory 26321e9f767SBen Widawsky * devices, and perhaps other types of CXL devices may have further information 26421e9f767SBen Widawsky * available upon error conditions. Driver facilities wishing to send mailbox 26521e9f767SBen Widawsky * commands should use the wrapper command. 26621e9f767SBen Widawsky * 26721e9f767SBen Widawsky * The CXL spec allows for up to two mailboxes. The intention is for the primary 26821e9f767SBen Widawsky * mailbox to be OS controlled and the secondary mailbox to be used by system 26921e9f767SBen Widawsky * firmware. This allows the OS and firmware to communicate with the device and 27021e9f767SBen Widawsky * not need to coordinate with each other. The driver only uses the primary 27121e9f767SBen Widawsky * mailbox. 27221e9f767SBen Widawsky */ 27321e9f767SBen Widawsky static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, 27421e9f767SBen Widawsky struct mbox_cmd *mbox_cmd) 27521e9f767SBen Widawsky { 27621e9f767SBen Widawsky void __iomem *payload = cxlm->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET; 27721e9f767SBen Widawsky u64 cmd_reg, status_reg; 27821e9f767SBen Widawsky size_t out_len; 27921e9f767SBen Widawsky int rc; 28021e9f767SBen Widawsky 28121e9f767SBen Widawsky lockdep_assert_held(&cxlm->mbox_mutex); 28221e9f767SBen Widawsky 28321e9f767SBen Widawsky /* 28421e9f767SBen Widawsky * Here are the steps from 8.2.8.4 of the CXL 2.0 spec. 28521e9f767SBen Widawsky * 1. Caller reads MB Control Register to verify doorbell is clear 28621e9f767SBen Widawsky * 2. Caller writes Command Register 28721e9f767SBen Widawsky * 3. Caller writes Command Payload Registers if input payload is non-empty 28821e9f767SBen Widawsky * 4. Caller writes MB Control Register to set doorbell 28921e9f767SBen Widawsky * 5. Caller either polls for doorbell to be clear or waits for interrupt if configured 29021e9f767SBen Widawsky * 6. Caller reads MB Status Register to fetch Return code 29121e9f767SBen Widawsky * 7. If command successful, Caller reads Command Register to get Payload Length 29221e9f767SBen Widawsky * 8. If output payload is non-empty, host reads Command Payload Registers 29321e9f767SBen Widawsky * 29421e9f767SBen Widawsky * Hardware is free to do whatever it wants before the doorbell is rung, 29521e9f767SBen Widawsky * and isn't allowed to change anything after it clears the doorbell. As 29621e9f767SBen Widawsky * such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can 29721e9f767SBen Widawsky * also happen in any order (though some orders might not make sense). 29821e9f767SBen Widawsky */ 29921e9f767SBen Widawsky 30021e9f767SBen Widawsky /* #1 */ 30121e9f767SBen Widawsky if (cxl_doorbell_busy(cxlm)) { 30221e9f767SBen Widawsky dev_err_ratelimited(&cxlm->pdev->dev, 30321e9f767SBen Widawsky "Mailbox re-busy after acquiring\n"); 30421e9f767SBen Widawsky return -EBUSY; 30521e9f767SBen Widawsky } 30621e9f767SBen Widawsky 30721e9f767SBen Widawsky cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK, 30821e9f767SBen Widawsky mbox_cmd->opcode); 30921e9f767SBen Widawsky if (mbox_cmd->size_in) { 31021e9f767SBen Widawsky if (WARN_ON(!mbox_cmd->payload_in)) 31121e9f767SBen Widawsky return -EINVAL; 31221e9f767SBen Widawsky 31321e9f767SBen Widawsky cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, 31421e9f767SBen Widawsky mbox_cmd->size_in); 31521e9f767SBen Widawsky memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in); 31621e9f767SBen Widawsky } 31721e9f767SBen Widawsky 31821e9f767SBen Widawsky /* #2, #3 */ 31921e9f767SBen Widawsky writeq(cmd_reg, cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 32021e9f767SBen Widawsky 32121e9f767SBen Widawsky /* #4 */ 32221e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, "Sending command\n"); 32321e9f767SBen Widawsky writel(CXLDEV_MBOX_CTRL_DOORBELL, 32421e9f767SBen Widawsky cxlm->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET); 32521e9f767SBen Widawsky 32621e9f767SBen Widawsky /* #5 */ 32721e9f767SBen Widawsky rc = cxl_mem_wait_for_doorbell(cxlm); 32821e9f767SBen Widawsky if (rc == -ETIMEDOUT) { 32921e9f767SBen Widawsky cxl_mem_mbox_timeout(cxlm, mbox_cmd); 33021e9f767SBen Widawsky return rc; 33121e9f767SBen Widawsky } 33221e9f767SBen Widawsky 33321e9f767SBen Widawsky /* #6 */ 33421e9f767SBen Widawsky status_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET); 33521e9f767SBen Widawsky mbox_cmd->return_code = 33621e9f767SBen Widawsky FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg); 33721e9f767SBen Widawsky 33821e9f767SBen Widawsky if (mbox_cmd->return_code != 0) { 33921e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, "Mailbox operation had an error\n"); 34021e9f767SBen Widawsky return 0; 34121e9f767SBen Widawsky } 34221e9f767SBen Widawsky 34321e9f767SBen Widawsky /* #7 */ 34421e9f767SBen Widawsky cmd_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 34521e9f767SBen Widawsky out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg); 34621e9f767SBen Widawsky 34721e9f767SBen Widawsky /* #8 */ 34821e9f767SBen Widawsky if (out_len && mbox_cmd->payload_out) { 34921e9f767SBen Widawsky /* 35021e9f767SBen Widawsky * Sanitize the copy. If hardware misbehaves, out_len per the 35121e9f767SBen Widawsky * spec can actually be greater than the max allowed size (21 35221e9f767SBen Widawsky * bits available but spec defined 1M max). The caller also may 35321e9f767SBen Widawsky * have requested less data than the hardware supplied even 35421e9f767SBen Widawsky * within spec. 35521e9f767SBen Widawsky */ 35621e9f767SBen Widawsky size_t n = min3(mbox_cmd->size_out, cxlm->payload_size, out_len); 35721e9f767SBen Widawsky 35821e9f767SBen Widawsky memcpy_fromio(mbox_cmd->payload_out, payload, n); 35921e9f767SBen Widawsky mbox_cmd->size_out = n; 36021e9f767SBen Widawsky } else { 36121e9f767SBen Widawsky mbox_cmd->size_out = 0; 36221e9f767SBen Widawsky } 36321e9f767SBen Widawsky 36421e9f767SBen Widawsky return 0; 36521e9f767SBen Widawsky } 36621e9f767SBen Widawsky 36721e9f767SBen Widawsky /** 36821e9f767SBen Widawsky * cxl_mem_mbox_get() - Acquire exclusive access to the mailbox. 36921e9f767SBen Widawsky * @cxlm: The memory device to gain access to. 37021e9f767SBen Widawsky * 37121e9f767SBen Widawsky * Context: Any context. Takes the mbox_mutex. 37221e9f767SBen Widawsky * Return: 0 if exclusive access was acquired. 37321e9f767SBen Widawsky */ 37421e9f767SBen Widawsky static int cxl_mem_mbox_get(struct cxl_mem *cxlm) 37521e9f767SBen Widawsky { 37621e9f767SBen Widawsky struct device *dev = &cxlm->pdev->dev; 37721e9f767SBen Widawsky u64 md_status; 37821e9f767SBen Widawsky int rc; 37921e9f767SBen Widawsky 38021e9f767SBen Widawsky mutex_lock_io(&cxlm->mbox_mutex); 38121e9f767SBen Widawsky 38221e9f767SBen Widawsky /* 38321e9f767SBen Widawsky * XXX: There is some amount of ambiguity in the 2.0 version of the spec 38421e9f767SBen Widawsky * around the mailbox interface ready (8.2.8.5.1.1). The purpose of the 38521e9f767SBen Widawsky * bit is to allow firmware running on the device to notify the driver 38621e9f767SBen Widawsky * that it's ready to receive commands. It is unclear if the bit needs 38721e9f767SBen Widawsky * to be read for each transaction mailbox, ie. the firmware can switch 38821e9f767SBen Widawsky * it on and off as needed. Second, there is no defined timeout for 38921e9f767SBen Widawsky * mailbox ready, like there is for the doorbell interface. 39021e9f767SBen Widawsky * 39121e9f767SBen Widawsky * Assumptions: 39221e9f767SBen Widawsky * 1. The firmware might toggle the Mailbox Interface Ready bit, check 39321e9f767SBen Widawsky * it for every command. 39421e9f767SBen Widawsky * 39521e9f767SBen Widawsky * 2. If the doorbell is clear, the firmware should have first set the 39621e9f767SBen Widawsky * Mailbox Interface Ready bit. Therefore, waiting for the doorbell 39721e9f767SBen Widawsky * to be ready is sufficient. 39821e9f767SBen Widawsky */ 39921e9f767SBen Widawsky rc = cxl_mem_wait_for_doorbell(cxlm); 40021e9f767SBen Widawsky if (rc) { 40121e9f767SBen Widawsky dev_warn(dev, "Mailbox interface not ready\n"); 40221e9f767SBen Widawsky goto out; 40321e9f767SBen Widawsky } 40421e9f767SBen Widawsky 40521e9f767SBen Widawsky md_status = readq(cxlm->regs.memdev + CXLMDEV_STATUS_OFFSET); 40621e9f767SBen Widawsky if (!(md_status & CXLMDEV_MBOX_IF_READY && CXLMDEV_READY(md_status))) { 40721e9f767SBen Widawsky dev_err(dev, "mbox: reported doorbell ready, but not mbox ready\n"); 40821e9f767SBen Widawsky rc = -EBUSY; 40921e9f767SBen Widawsky goto out; 41021e9f767SBen Widawsky } 41121e9f767SBen Widawsky 41221e9f767SBen Widawsky /* 41321e9f767SBen Widawsky * Hardware shouldn't allow a ready status but also have failure bits 41421e9f767SBen Widawsky * set. Spit out an error, this should be a bug report 41521e9f767SBen Widawsky */ 41621e9f767SBen Widawsky rc = -EFAULT; 41721e9f767SBen Widawsky if (md_status & CXLMDEV_DEV_FATAL) { 41821e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but fatal\n"); 41921e9f767SBen Widawsky goto out; 42021e9f767SBen Widawsky } 42121e9f767SBen Widawsky if (md_status & CXLMDEV_FW_HALT) { 42221e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but halted\n"); 42321e9f767SBen Widawsky goto out; 42421e9f767SBen Widawsky } 42521e9f767SBen Widawsky if (CXLMDEV_RESET_NEEDED(md_status)) { 42621e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but reset needed\n"); 42721e9f767SBen Widawsky goto out; 42821e9f767SBen Widawsky } 42921e9f767SBen Widawsky 43021e9f767SBen Widawsky /* with lock held */ 43121e9f767SBen Widawsky return 0; 43221e9f767SBen Widawsky 43321e9f767SBen Widawsky out: 43421e9f767SBen Widawsky mutex_unlock(&cxlm->mbox_mutex); 43521e9f767SBen Widawsky return rc; 43621e9f767SBen Widawsky } 43721e9f767SBen Widawsky 43821e9f767SBen Widawsky /** 43921e9f767SBen Widawsky * cxl_mem_mbox_put() - Release exclusive access to the mailbox. 44021e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 44121e9f767SBen Widawsky * 44221e9f767SBen Widawsky * Context: Any context. Expects mbox_mutex to be held. 44321e9f767SBen Widawsky */ 44421e9f767SBen Widawsky static void cxl_mem_mbox_put(struct cxl_mem *cxlm) 44521e9f767SBen Widawsky { 44621e9f767SBen Widawsky mutex_unlock(&cxlm->mbox_mutex); 44721e9f767SBen Widawsky } 44821e9f767SBen Widawsky 44921e9f767SBen Widawsky /** 45021e9f767SBen Widawsky * handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace. 45121e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 45221e9f767SBen Widawsky * @cmd: The validated command. 45321e9f767SBen Widawsky * @in_payload: Pointer to userspace's input payload. 45421e9f767SBen Widawsky * @out_payload: Pointer to userspace's output payload. 45521e9f767SBen Widawsky * @size_out: (Input) Max payload size to copy out. 45621e9f767SBen Widawsky * (Output) Payload size hardware generated. 45721e9f767SBen Widawsky * @retval: Hardware generated return code from the operation. 45821e9f767SBen Widawsky * 45921e9f767SBen Widawsky * Return: 46021e9f767SBen Widawsky * * %0 - Mailbox transaction succeeded. This implies the mailbox 46121e9f767SBen Widawsky * protocol completed successfully not that the operation itself 46221e9f767SBen Widawsky * was successful. 46321e9f767SBen Widawsky * * %-ENOMEM - Couldn't allocate a bounce buffer. 46421e9f767SBen Widawsky * * %-EFAULT - Something happened with copy_to/from_user. 46521e9f767SBen Widawsky * * %-EINTR - Mailbox acquisition interrupted. 46621e9f767SBen Widawsky * * %-EXXX - Transaction level failures. 46721e9f767SBen Widawsky * 46821e9f767SBen Widawsky * Creates the appropriate mailbox command and dispatches it on behalf of a 46921e9f767SBen Widawsky * userspace request. The input and output payloads are copied between 47021e9f767SBen Widawsky * userspace. 47121e9f767SBen Widawsky * 47221e9f767SBen Widawsky * See cxl_send_cmd(). 47321e9f767SBen Widawsky */ 47421e9f767SBen Widawsky static int handle_mailbox_cmd_from_user(struct cxl_mem *cxlm, 47521e9f767SBen Widawsky const struct cxl_mem_command *cmd, 47621e9f767SBen Widawsky u64 in_payload, u64 out_payload, 47721e9f767SBen Widawsky s32 *size_out, u32 *retval) 47821e9f767SBen Widawsky { 47921e9f767SBen Widawsky struct device *dev = &cxlm->pdev->dev; 48021e9f767SBen Widawsky struct mbox_cmd mbox_cmd = { 48121e9f767SBen Widawsky .opcode = cmd->opcode, 48221e9f767SBen Widawsky .size_in = cmd->info.size_in, 48321e9f767SBen Widawsky .size_out = cmd->info.size_out, 48421e9f767SBen Widawsky }; 48521e9f767SBen Widawsky int rc; 48621e9f767SBen Widawsky 48721e9f767SBen Widawsky if (cmd->info.size_out) { 48821e9f767SBen Widawsky mbox_cmd.payload_out = kvzalloc(cmd->info.size_out, GFP_KERNEL); 48921e9f767SBen Widawsky if (!mbox_cmd.payload_out) 49021e9f767SBen Widawsky return -ENOMEM; 49121e9f767SBen Widawsky } 49221e9f767SBen Widawsky 49321e9f767SBen Widawsky if (cmd->info.size_in) { 49421e9f767SBen Widawsky mbox_cmd.payload_in = vmemdup_user(u64_to_user_ptr(in_payload), 49521e9f767SBen Widawsky cmd->info.size_in); 49621e9f767SBen Widawsky if (IS_ERR(mbox_cmd.payload_in)) { 49721e9f767SBen Widawsky kvfree(mbox_cmd.payload_out); 49821e9f767SBen Widawsky return PTR_ERR(mbox_cmd.payload_in); 49921e9f767SBen Widawsky } 50021e9f767SBen Widawsky } 50121e9f767SBen Widawsky 50221e9f767SBen Widawsky rc = cxl_mem_mbox_get(cxlm); 50321e9f767SBen Widawsky if (rc) 50421e9f767SBen Widawsky goto out; 50521e9f767SBen Widawsky 50621e9f767SBen Widawsky dev_dbg(dev, 50721e9f767SBen Widawsky "Submitting %s command for user\n" 50821e9f767SBen Widawsky "\topcode: %x\n" 50921e9f767SBen Widawsky "\tsize: %ub\n", 51021e9f767SBen Widawsky cxl_command_names[cmd->info.id].name, mbox_cmd.opcode, 51121e9f767SBen Widawsky cmd->info.size_in); 51221e9f767SBen Widawsky 51321e9f767SBen Widawsky dev_WARN_ONCE(dev, cmd->info.id == CXL_MEM_COMMAND_ID_RAW, 51421e9f767SBen Widawsky "raw command path used\n"); 51521e9f767SBen Widawsky 51621e9f767SBen Widawsky rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd); 51721e9f767SBen Widawsky cxl_mem_mbox_put(cxlm); 51821e9f767SBen Widawsky if (rc) 51921e9f767SBen Widawsky goto out; 52021e9f767SBen Widawsky 52121e9f767SBen Widawsky /* 52221e9f767SBen Widawsky * @size_out contains the max size that's allowed to be written back out 52321e9f767SBen Widawsky * to userspace. While the payload may have written more output than 52421e9f767SBen Widawsky * this it will have to be ignored. 52521e9f767SBen Widawsky */ 52621e9f767SBen Widawsky if (mbox_cmd.size_out) { 52721e9f767SBen Widawsky dev_WARN_ONCE(dev, mbox_cmd.size_out > *size_out, 52821e9f767SBen Widawsky "Invalid return size\n"); 52921e9f767SBen Widawsky if (copy_to_user(u64_to_user_ptr(out_payload), 53021e9f767SBen Widawsky mbox_cmd.payload_out, mbox_cmd.size_out)) { 53121e9f767SBen Widawsky rc = -EFAULT; 53221e9f767SBen Widawsky goto out; 53321e9f767SBen Widawsky } 53421e9f767SBen Widawsky } 53521e9f767SBen Widawsky 53621e9f767SBen Widawsky *size_out = mbox_cmd.size_out; 53721e9f767SBen Widawsky *retval = mbox_cmd.return_code; 53821e9f767SBen Widawsky 53921e9f767SBen Widawsky out: 54021e9f767SBen Widawsky kvfree(mbox_cmd.payload_in); 54121e9f767SBen Widawsky kvfree(mbox_cmd.payload_out); 54221e9f767SBen Widawsky return rc; 54321e9f767SBen Widawsky } 54421e9f767SBen Widawsky 54521e9f767SBen Widawsky static bool cxl_mem_raw_command_allowed(u16 opcode) 54621e9f767SBen Widawsky { 54721e9f767SBen Widawsky int i; 54821e9f767SBen Widawsky 54921e9f767SBen Widawsky if (!IS_ENABLED(CONFIG_CXL_MEM_RAW_COMMANDS)) 55021e9f767SBen Widawsky return false; 55121e9f767SBen Widawsky 55221e9f767SBen Widawsky if (security_locked_down(LOCKDOWN_NONE)) 55321e9f767SBen Widawsky return false; 55421e9f767SBen Widawsky 55521e9f767SBen Widawsky if (cxl_raw_allow_all) 55621e9f767SBen Widawsky return true; 55721e9f767SBen Widawsky 55821e9f767SBen Widawsky if (cxl_is_security_command(opcode)) 55921e9f767SBen Widawsky return false; 56021e9f767SBen Widawsky 56121e9f767SBen Widawsky for (i = 0; i < ARRAY_SIZE(cxl_disabled_raw_commands); i++) 56221e9f767SBen Widawsky if (cxl_disabled_raw_commands[i] == opcode) 56321e9f767SBen Widawsky return false; 56421e9f767SBen Widawsky 56521e9f767SBen Widawsky return true; 56621e9f767SBen Widawsky } 56721e9f767SBen Widawsky 56821e9f767SBen Widawsky /** 56921e9f767SBen Widawsky * cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND. 57021e9f767SBen Widawsky * @cxlm: &struct cxl_mem device whose mailbox will be used. 57121e9f767SBen Widawsky * @send_cmd: &struct cxl_send_command copied in from userspace. 57221e9f767SBen Widawsky * @out_cmd: Sanitized and populated &struct cxl_mem_command. 57321e9f767SBen Widawsky * 57421e9f767SBen Widawsky * Return: 57521e9f767SBen Widawsky * * %0 - @out_cmd is ready to send. 57621e9f767SBen Widawsky * * %-ENOTTY - Invalid command specified. 57721e9f767SBen Widawsky * * %-EINVAL - Reserved fields or invalid values were used. 57821e9f767SBen Widawsky * * %-ENOMEM - Input or output buffer wasn't sized properly. 57921e9f767SBen Widawsky * * %-EPERM - Attempted to use a protected command. 58021e9f767SBen Widawsky * 58121e9f767SBen Widawsky * The result of this command is a fully validated command in @out_cmd that is 58221e9f767SBen Widawsky * safe to send to the hardware. 58321e9f767SBen Widawsky * 58421e9f767SBen Widawsky * See handle_mailbox_cmd_from_user() 58521e9f767SBen Widawsky */ 58621e9f767SBen Widawsky static int cxl_validate_cmd_from_user(struct cxl_mem *cxlm, 58721e9f767SBen Widawsky const struct cxl_send_command *send_cmd, 58821e9f767SBen Widawsky struct cxl_mem_command *out_cmd) 58921e9f767SBen Widawsky { 59021e9f767SBen Widawsky const struct cxl_command_info *info; 59121e9f767SBen Widawsky struct cxl_mem_command *c; 59221e9f767SBen Widawsky 59321e9f767SBen Widawsky if (send_cmd->id == 0 || send_cmd->id >= CXL_MEM_COMMAND_ID_MAX) 59421e9f767SBen Widawsky return -ENOTTY; 59521e9f767SBen Widawsky 59621e9f767SBen Widawsky /* 59721e9f767SBen Widawsky * The user can never specify an input payload larger than what hardware 59821e9f767SBen Widawsky * supports, but output can be arbitrarily large (simply write out as 59921e9f767SBen Widawsky * much data as the hardware provides). 60021e9f767SBen Widawsky */ 60121e9f767SBen Widawsky if (send_cmd->in.size > cxlm->payload_size) 60221e9f767SBen Widawsky return -EINVAL; 60321e9f767SBen Widawsky 60421e9f767SBen Widawsky /* 60521e9f767SBen Widawsky * Checks are bypassed for raw commands but a WARN/taint will occur 60621e9f767SBen Widawsky * later in the callchain 60721e9f767SBen Widawsky */ 60821e9f767SBen Widawsky if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW) { 60921e9f767SBen Widawsky const struct cxl_mem_command temp = { 61021e9f767SBen Widawsky .info = { 61121e9f767SBen Widawsky .id = CXL_MEM_COMMAND_ID_RAW, 61221e9f767SBen Widawsky .flags = 0, 61321e9f767SBen Widawsky .size_in = send_cmd->in.size, 61421e9f767SBen Widawsky .size_out = send_cmd->out.size, 61521e9f767SBen Widawsky }, 61621e9f767SBen Widawsky .opcode = send_cmd->raw.opcode 61721e9f767SBen Widawsky }; 61821e9f767SBen Widawsky 61921e9f767SBen Widawsky if (send_cmd->raw.rsvd) 62021e9f767SBen Widawsky return -EINVAL; 62121e9f767SBen Widawsky 62221e9f767SBen Widawsky /* 62321e9f767SBen Widawsky * Unlike supported commands, the output size of RAW commands 62421e9f767SBen Widawsky * gets passed along without further checking, so it must be 62521e9f767SBen Widawsky * validated here. 62621e9f767SBen Widawsky */ 62721e9f767SBen Widawsky if (send_cmd->out.size > cxlm->payload_size) 62821e9f767SBen Widawsky return -EINVAL; 62921e9f767SBen Widawsky 63021e9f767SBen Widawsky if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode)) 63121e9f767SBen Widawsky return -EPERM; 63221e9f767SBen Widawsky 63321e9f767SBen Widawsky memcpy(out_cmd, &temp, sizeof(temp)); 63421e9f767SBen Widawsky 63521e9f767SBen Widawsky return 0; 63621e9f767SBen Widawsky } 63721e9f767SBen Widawsky 63821e9f767SBen Widawsky if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK) 63921e9f767SBen Widawsky return -EINVAL; 64021e9f767SBen Widawsky 64121e9f767SBen Widawsky if (send_cmd->rsvd) 64221e9f767SBen Widawsky return -EINVAL; 64321e9f767SBen Widawsky 64421e9f767SBen Widawsky if (send_cmd->in.rsvd || send_cmd->out.rsvd) 64521e9f767SBen Widawsky return -EINVAL; 64621e9f767SBen Widawsky 64721e9f767SBen Widawsky /* Convert user's command into the internal representation */ 64821e9f767SBen Widawsky c = &mem_commands[send_cmd->id]; 64921e9f767SBen Widawsky info = &c->info; 65021e9f767SBen Widawsky 65121e9f767SBen Widawsky /* Check that the command is enabled for hardware */ 65221e9f767SBen Widawsky if (!test_bit(info->id, cxlm->enabled_cmds)) 65321e9f767SBen Widawsky return -ENOTTY; 65421e9f767SBen Widawsky 65521e9f767SBen Widawsky /* Check the input buffer is the expected size */ 65621e9f767SBen Widawsky if (info->size_in >= 0 && info->size_in != send_cmd->in.size) 65721e9f767SBen Widawsky return -ENOMEM; 65821e9f767SBen Widawsky 65921e9f767SBen Widawsky /* Check the output buffer is at least large enough */ 66021e9f767SBen Widawsky if (info->size_out >= 0 && send_cmd->out.size < info->size_out) 66121e9f767SBen Widawsky return -ENOMEM; 66221e9f767SBen Widawsky 66321e9f767SBen Widawsky memcpy(out_cmd, c, sizeof(*c)); 66421e9f767SBen Widawsky out_cmd->info.size_in = send_cmd->in.size; 66521e9f767SBen Widawsky /* 66621e9f767SBen Widawsky * XXX: out_cmd->info.size_out will be controlled by the driver, and the 66721e9f767SBen Widawsky * specified number of bytes @send_cmd->out.size will be copied back out 66821e9f767SBen Widawsky * to userspace. 66921e9f767SBen Widawsky */ 67021e9f767SBen Widawsky 67121e9f767SBen Widawsky return 0; 67221e9f767SBen Widawsky } 67321e9f767SBen Widawsky 67421e9f767SBen Widawsky static int cxl_query_cmd(struct cxl_memdev *cxlmd, 67521e9f767SBen Widawsky struct cxl_mem_query_commands __user *q) 67621e9f767SBen Widawsky { 67721e9f767SBen Widawsky struct device *dev = &cxlmd->dev; 67821e9f767SBen Widawsky struct cxl_mem_command *cmd; 67921e9f767SBen Widawsky u32 n_commands; 68021e9f767SBen Widawsky int j = 0; 68121e9f767SBen Widawsky 68221e9f767SBen Widawsky dev_dbg(dev, "Query IOCTL\n"); 68321e9f767SBen Widawsky 68421e9f767SBen Widawsky if (get_user(n_commands, &q->n_commands)) 68521e9f767SBen Widawsky return -EFAULT; 68621e9f767SBen Widawsky 68721e9f767SBen Widawsky /* returns the total number if 0 elements are requested. */ 68821e9f767SBen Widawsky if (n_commands == 0) 68921e9f767SBen Widawsky return put_user(cxl_cmd_count, &q->n_commands); 69021e9f767SBen Widawsky 69121e9f767SBen Widawsky /* 69221e9f767SBen Widawsky * otherwise, return max(n_commands, total commands) cxl_command_info 69321e9f767SBen Widawsky * structures. 69421e9f767SBen Widawsky */ 69521e9f767SBen Widawsky cxl_for_each_cmd(cmd) { 69621e9f767SBen Widawsky const struct cxl_command_info *info = &cmd->info; 69721e9f767SBen Widawsky 69821e9f767SBen Widawsky if (copy_to_user(&q->commands[j++], info, sizeof(*info))) 69921e9f767SBen Widawsky return -EFAULT; 70021e9f767SBen Widawsky 70121e9f767SBen Widawsky if (j == n_commands) 70221e9f767SBen Widawsky break; 70321e9f767SBen Widawsky } 70421e9f767SBen Widawsky 70521e9f767SBen Widawsky return 0; 70621e9f767SBen Widawsky } 70721e9f767SBen Widawsky 70821e9f767SBen Widawsky static int cxl_send_cmd(struct cxl_memdev *cxlmd, 70921e9f767SBen Widawsky struct cxl_send_command __user *s) 71021e9f767SBen Widawsky { 71121e9f767SBen Widawsky struct cxl_mem *cxlm = cxlmd->cxlm; 71221e9f767SBen Widawsky struct device *dev = &cxlmd->dev; 71321e9f767SBen Widawsky struct cxl_send_command send; 71421e9f767SBen Widawsky struct cxl_mem_command c; 71521e9f767SBen Widawsky int rc; 71621e9f767SBen Widawsky 71721e9f767SBen Widawsky dev_dbg(dev, "Send IOCTL\n"); 71821e9f767SBen Widawsky 71921e9f767SBen Widawsky if (copy_from_user(&send, s, sizeof(send))) 72021e9f767SBen Widawsky return -EFAULT; 72121e9f767SBen Widawsky 72221e9f767SBen Widawsky rc = cxl_validate_cmd_from_user(cxlmd->cxlm, &send, &c); 72321e9f767SBen Widawsky if (rc) 72421e9f767SBen Widawsky return rc; 72521e9f767SBen Widawsky 72621e9f767SBen Widawsky /* Prepare to handle a full payload for variable sized output */ 72721e9f767SBen Widawsky if (c.info.size_out < 0) 72821e9f767SBen Widawsky c.info.size_out = cxlm->payload_size; 72921e9f767SBen Widawsky 73021e9f767SBen Widawsky rc = handle_mailbox_cmd_from_user(cxlm, &c, send.in.payload, 73121e9f767SBen Widawsky send.out.payload, &send.out.size, 73221e9f767SBen Widawsky &send.retval); 73321e9f767SBen Widawsky if (rc) 73421e9f767SBen Widawsky return rc; 73521e9f767SBen Widawsky 73621e9f767SBen Widawsky if (copy_to_user(s, &send, sizeof(send))) 73721e9f767SBen Widawsky return -EFAULT; 73821e9f767SBen Widawsky 73921e9f767SBen Widawsky return 0; 74021e9f767SBen Widawsky } 74121e9f767SBen Widawsky 74221e9f767SBen Widawsky static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, 74321e9f767SBen Widawsky unsigned long arg) 74421e9f767SBen Widawsky { 74521e9f767SBen Widawsky switch (cmd) { 74621e9f767SBen Widawsky case CXL_MEM_QUERY_COMMANDS: 74721e9f767SBen Widawsky return cxl_query_cmd(cxlmd, (void __user *)arg); 74821e9f767SBen Widawsky case CXL_MEM_SEND_COMMAND: 74921e9f767SBen Widawsky return cxl_send_cmd(cxlmd, (void __user *)arg); 75021e9f767SBen Widawsky default: 75121e9f767SBen Widawsky return -ENOTTY; 75221e9f767SBen Widawsky } 75321e9f767SBen Widawsky } 75421e9f767SBen Widawsky 75521e9f767SBen Widawsky static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, 75621e9f767SBen Widawsky unsigned long arg) 75721e9f767SBen Widawsky { 75821e9f767SBen Widawsky struct cxl_memdev *cxlmd = file->private_data; 75921e9f767SBen Widawsky int rc = -ENXIO; 76021e9f767SBen Widawsky 76121e9f767SBen Widawsky down_read(&cxl_memdev_rwsem); 76221e9f767SBen Widawsky if (cxlmd->cxlm) 76321e9f767SBen Widawsky rc = __cxl_memdev_ioctl(cxlmd, cmd, arg); 76421e9f767SBen Widawsky up_read(&cxl_memdev_rwsem); 76521e9f767SBen Widawsky 76621e9f767SBen Widawsky return rc; 76721e9f767SBen Widawsky } 76821e9f767SBen Widawsky 76921e9f767SBen Widawsky static int cxl_memdev_open(struct inode *inode, struct file *file) 77021e9f767SBen Widawsky { 77121e9f767SBen Widawsky struct cxl_memdev *cxlmd = 77221e9f767SBen Widawsky container_of(inode->i_cdev, typeof(*cxlmd), cdev); 77321e9f767SBen Widawsky 77421e9f767SBen Widawsky get_device(&cxlmd->dev); 77521e9f767SBen Widawsky file->private_data = cxlmd; 77621e9f767SBen Widawsky 77721e9f767SBen Widawsky return 0; 77821e9f767SBen Widawsky } 77921e9f767SBen Widawsky 78021e9f767SBen Widawsky static int cxl_memdev_release_file(struct inode *inode, struct file *file) 78121e9f767SBen Widawsky { 78221e9f767SBen Widawsky struct cxl_memdev *cxlmd = 78321e9f767SBen Widawsky container_of(inode->i_cdev, typeof(*cxlmd), cdev); 78421e9f767SBen Widawsky 78521e9f767SBen Widawsky put_device(&cxlmd->dev); 78621e9f767SBen Widawsky 78721e9f767SBen Widawsky return 0; 78821e9f767SBen Widawsky } 78921e9f767SBen Widawsky 79021e9f767SBen Widawsky static const struct file_operations cxl_memdev_fops = { 79121e9f767SBen Widawsky .owner = THIS_MODULE, 79221e9f767SBen Widawsky .unlocked_ioctl = cxl_memdev_ioctl, 79321e9f767SBen Widawsky .open = cxl_memdev_open, 79421e9f767SBen Widawsky .release = cxl_memdev_release_file, 79521e9f767SBen Widawsky .compat_ioctl = compat_ptr_ioctl, 79621e9f767SBen Widawsky .llseek = noop_llseek, 79721e9f767SBen Widawsky }; 79821e9f767SBen Widawsky 79921e9f767SBen Widawsky static inline struct cxl_mem_command *cxl_mem_find_command(u16 opcode) 80021e9f767SBen Widawsky { 80121e9f767SBen Widawsky struct cxl_mem_command *c; 80221e9f767SBen Widawsky 80321e9f767SBen Widawsky cxl_for_each_cmd(c) 80421e9f767SBen Widawsky if (c->opcode == opcode) 80521e9f767SBen Widawsky return c; 80621e9f767SBen Widawsky 80721e9f767SBen Widawsky return NULL; 80821e9f767SBen Widawsky } 80921e9f767SBen Widawsky 81021e9f767SBen Widawsky /** 81121e9f767SBen Widawsky * cxl_mem_mbox_send_cmd() - Send a mailbox command to a memory device. 81221e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 81321e9f767SBen Widawsky * @opcode: Opcode for the mailbox command. 81421e9f767SBen Widawsky * @in: The input payload for the mailbox command. 81521e9f767SBen Widawsky * @in_size: The length of the input payload 81621e9f767SBen Widawsky * @out: Caller allocated buffer for the output. 81721e9f767SBen Widawsky * @out_size: Expected size of output. 81821e9f767SBen Widawsky * 81921e9f767SBen Widawsky * Context: Any context. Will acquire and release mbox_mutex. 82021e9f767SBen Widawsky * Return: 82121e9f767SBen Widawsky * * %>=0 - Number of bytes returned in @out. 82221e9f767SBen Widawsky * * %-E2BIG - Payload is too large for hardware. 82321e9f767SBen Widawsky * * %-EBUSY - Couldn't acquire exclusive mailbox access. 82421e9f767SBen Widawsky * * %-EFAULT - Hardware error occurred. 82521e9f767SBen Widawsky * * %-ENXIO - Command completed, but device reported an error. 82621e9f767SBen Widawsky * * %-EIO - Unexpected output size. 82721e9f767SBen Widawsky * 82821e9f767SBen Widawsky * Mailbox commands may execute successfully yet the device itself reported an 82921e9f767SBen Widawsky * error. While this distinction can be useful for commands from userspace, the 83021e9f767SBen Widawsky * kernel will only be able to use results when both are successful. 83121e9f767SBen Widawsky * 83221e9f767SBen Widawsky * See __cxl_mem_mbox_send_cmd() 83321e9f767SBen Widawsky */ 83421e9f767SBen Widawsky static int cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, u16 opcode, 83521e9f767SBen Widawsky void *in, size_t in_size, 83621e9f767SBen Widawsky void *out, size_t out_size) 83721e9f767SBen Widawsky { 83821e9f767SBen Widawsky const struct cxl_mem_command *cmd = cxl_mem_find_command(opcode); 83921e9f767SBen Widawsky struct mbox_cmd mbox_cmd = { 84021e9f767SBen Widawsky .opcode = opcode, 84121e9f767SBen Widawsky .payload_in = in, 84221e9f767SBen Widawsky .size_in = in_size, 84321e9f767SBen Widawsky .size_out = out_size, 84421e9f767SBen Widawsky .payload_out = out, 84521e9f767SBen Widawsky }; 84621e9f767SBen Widawsky int rc; 84721e9f767SBen Widawsky 84821e9f767SBen Widawsky if (out_size > cxlm->payload_size) 84921e9f767SBen Widawsky return -E2BIG; 85021e9f767SBen Widawsky 85121e9f767SBen Widawsky rc = cxl_mem_mbox_get(cxlm); 85221e9f767SBen Widawsky if (rc) 85321e9f767SBen Widawsky return rc; 85421e9f767SBen Widawsky 85521e9f767SBen Widawsky rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd); 85621e9f767SBen Widawsky cxl_mem_mbox_put(cxlm); 85721e9f767SBen Widawsky if (rc) 85821e9f767SBen Widawsky return rc; 85921e9f767SBen Widawsky 86021e9f767SBen Widawsky /* TODO: Map return code to proper kernel style errno */ 86121e9f767SBen Widawsky if (mbox_cmd.return_code != CXL_MBOX_SUCCESS) 86221e9f767SBen Widawsky return -ENXIO; 86321e9f767SBen Widawsky 86421e9f767SBen Widawsky /* 86521e9f767SBen Widawsky * Variable sized commands can't be validated and so it's up to the 86621e9f767SBen Widawsky * caller to do that if they wish. 86721e9f767SBen Widawsky */ 86821e9f767SBen Widawsky if (cmd->info.size_out >= 0 && mbox_cmd.size_out != out_size) 86921e9f767SBen Widawsky return -EIO; 87021e9f767SBen Widawsky 87121e9f767SBen Widawsky return 0; 87221e9f767SBen Widawsky } 87321e9f767SBen Widawsky 87421e9f767SBen Widawsky static int cxl_mem_setup_mailbox(struct cxl_mem *cxlm) 87521e9f767SBen Widawsky { 87621e9f767SBen Widawsky const int cap = readl(cxlm->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET); 87721e9f767SBen Widawsky 87821e9f767SBen Widawsky cxlm->payload_size = 87921e9f767SBen Widawsky 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap); 88021e9f767SBen Widawsky 88121e9f767SBen Widawsky /* 88221e9f767SBen Widawsky * CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register 88321e9f767SBen Widawsky * 88421e9f767SBen Widawsky * If the size is too small, mandatory commands will not work and so 88521e9f767SBen Widawsky * there's no point in going forward. If the size is too large, there's 88621e9f767SBen Widawsky * no harm is soft limiting it. 88721e9f767SBen Widawsky */ 88821e9f767SBen Widawsky cxlm->payload_size = min_t(size_t, cxlm->payload_size, SZ_1M); 88921e9f767SBen Widawsky if (cxlm->payload_size < 256) { 89021e9f767SBen Widawsky dev_err(&cxlm->pdev->dev, "Mailbox is too small (%zub)", 89121e9f767SBen Widawsky cxlm->payload_size); 89221e9f767SBen Widawsky return -ENXIO; 89321e9f767SBen Widawsky } 89421e9f767SBen Widawsky 89521e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, "Mailbox payload sized %zu", 89621e9f767SBen Widawsky cxlm->payload_size); 89721e9f767SBen Widawsky 89821e9f767SBen Widawsky return 0; 89921e9f767SBen Widawsky } 90021e9f767SBen Widawsky 9011b0a1a2aSBen Widawsky static struct cxl_mem *cxl_mem_create(struct pci_dev *pdev) 90221e9f767SBen Widawsky { 90321e9f767SBen Widawsky struct device *dev = &pdev->dev; 90421e9f767SBen Widawsky struct cxl_mem *cxlm; 90521e9f767SBen Widawsky 9065d0c6f02SBen Widawsky cxlm = devm_kzalloc(dev, sizeof(*cxlm), GFP_KERNEL); 90721e9f767SBen Widawsky if (!cxlm) { 90821e9f767SBen Widawsky dev_err(dev, "No memory available\n"); 9091b0a1a2aSBen Widawsky return ERR_PTR(-ENOMEM); 91021e9f767SBen Widawsky } 91121e9f767SBen Widawsky 9121b0a1a2aSBen Widawsky mutex_init(&cxlm->mbox_mutex); 9131b0a1a2aSBen Widawsky cxlm->pdev = pdev; 9141b0a1a2aSBen Widawsky cxlm->enabled_cmds = 9151b0a1a2aSBen Widawsky devm_kmalloc_array(dev, BITS_TO_LONGS(cxl_cmd_count), 9161b0a1a2aSBen Widawsky sizeof(unsigned long), 9171b0a1a2aSBen Widawsky GFP_KERNEL | __GFP_ZERO); 9181b0a1a2aSBen Widawsky if (!cxlm->enabled_cmds) { 9191b0a1a2aSBen Widawsky dev_err(dev, "No memory available for bitmap\n"); 9201b0a1a2aSBen Widawsky return ERR_PTR(-ENOMEM); 9211b0a1a2aSBen Widawsky } 9221b0a1a2aSBen Widawsky 9231b0a1a2aSBen Widawsky return cxlm; 9241b0a1a2aSBen Widawsky } 9251b0a1a2aSBen Widawsky 92607d62eacSIra Weiny static void __iomem *cxl_mem_map_regblock(struct cxl_mem *cxlm, 92707d62eacSIra Weiny u8 bar, u64 offset) 9281b0a1a2aSBen Widawsky { 9291b0a1a2aSBen Widawsky struct pci_dev *pdev = cxlm->pdev; 9301b0a1a2aSBen Widawsky struct device *dev = &pdev->dev; 931f8a7e8c2SIra Weiny void __iomem *addr; 9321b0a1a2aSBen Widawsky 93321e9f767SBen Widawsky /* Basic sanity check that BAR is big enough */ 93421e9f767SBen Widawsky if (pci_resource_len(pdev, bar) < offset) { 93521e9f767SBen Widawsky dev_err(dev, "BAR%d: %pr: too small (offset: %#llx)\n", bar, 93621e9f767SBen Widawsky &pdev->resource[bar], (unsigned long long)offset); 9376630d31cSBen Widawsky return IOMEM_ERR_PTR(-ENXIO); 93821e9f767SBen Widawsky } 93921e9f767SBen Widawsky 94030af9729SIra Weiny addr = pci_iomap(pdev, bar, 0); 941f8a7e8c2SIra Weiny if (!addr) { 94221e9f767SBen Widawsky dev_err(dev, "failed to map registers\n"); 943f8a7e8c2SIra Weiny return addr; 94421e9f767SBen Widawsky } 94521e9f767SBen Widawsky 946f8a7e8c2SIra Weiny dev_dbg(dev, "Mapped CXL Memory Device resource bar %u @ %#llx\n", 947f8a7e8c2SIra Weiny bar, offset); 9486630d31cSBen Widawsky 94930af9729SIra Weiny return addr; 95030af9729SIra Weiny } 95130af9729SIra Weiny 95230af9729SIra Weiny static void cxl_mem_unmap_regblock(struct cxl_mem *cxlm, void __iomem *base) 95330af9729SIra Weiny { 95430af9729SIra Weiny pci_iounmap(cxlm->pdev, base); 95521e9f767SBen Widawsky } 95621e9f767SBen Widawsky 95721e9f767SBen Widawsky static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec) 95821e9f767SBen Widawsky { 95921e9f767SBen Widawsky int pos; 96021e9f767SBen Widawsky 96121e9f767SBen Widawsky pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DVSEC); 96221e9f767SBen Widawsky if (!pos) 96321e9f767SBen Widawsky return 0; 96421e9f767SBen Widawsky 96521e9f767SBen Widawsky while (pos) { 96621e9f767SBen Widawsky u16 vendor, id; 96721e9f767SBen Widawsky 96821e9f767SBen Widawsky pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER1, &vendor); 96921e9f767SBen Widawsky pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER2, &id); 97021e9f767SBen Widawsky if (vendor == PCI_DVSEC_VENDOR_ID_CXL && dvsec == id) 97121e9f767SBen Widawsky return pos; 97221e9f767SBen Widawsky 97321e9f767SBen Widawsky pos = pci_find_next_ext_capability(pdev, pos, 97421e9f767SBen Widawsky PCI_EXT_CAP_ID_DVSEC); 97521e9f767SBen Widawsky } 97621e9f767SBen Widawsky 97721e9f767SBen Widawsky return 0; 97821e9f767SBen Widawsky } 97921e9f767SBen Widawsky 98030af9729SIra Weiny static int cxl_probe_regs(struct cxl_mem *cxlm, void __iomem *base, 98130af9729SIra Weiny struct cxl_register_map *map) 98230af9729SIra Weiny { 98330af9729SIra Weiny struct pci_dev *pdev = cxlm->pdev; 98430af9729SIra Weiny struct device *dev = &pdev->dev; 98530af9729SIra Weiny struct cxl_device_reg_map *dev_map; 98630af9729SIra Weiny 98730af9729SIra Weiny switch (map->reg_type) { 98830af9729SIra Weiny case CXL_REGLOC_RBI_MEMDEV: 98930af9729SIra Weiny dev_map = &map->device_map; 99030af9729SIra Weiny cxl_probe_device_regs(dev, base, dev_map); 99130af9729SIra Weiny if (!dev_map->status.valid || !dev_map->mbox.valid || 99230af9729SIra Weiny !dev_map->memdev.valid) { 99330af9729SIra Weiny dev_err(dev, "registers not found: %s%s%s\n", 99430af9729SIra Weiny !dev_map->status.valid ? "status " : "", 99530af9729SIra Weiny !dev_map->mbox.valid ? "status " : "", 99630af9729SIra Weiny !dev_map->memdev.valid ? "status " : ""); 99730af9729SIra Weiny return -ENXIO; 99830af9729SIra Weiny } 99930af9729SIra Weiny 100030af9729SIra Weiny dev_dbg(dev, "Probing device registers...\n"); 100130af9729SIra Weiny break; 100230af9729SIra Weiny default: 100330af9729SIra Weiny break; 100430af9729SIra Weiny } 100530af9729SIra Weiny 100630af9729SIra Weiny return 0; 100730af9729SIra Weiny } 100830af9729SIra Weiny 100930af9729SIra Weiny static int cxl_map_regs(struct cxl_mem *cxlm, struct cxl_register_map *map) 101030af9729SIra Weiny { 101130af9729SIra Weiny struct pci_dev *pdev = cxlm->pdev; 101230af9729SIra Weiny struct device *dev = &pdev->dev; 101330af9729SIra Weiny 101430af9729SIra Weiny switch (map->reg_type) { 101530af9729SIra Weiny case CXL_REGLOC_RBI_MEMDEV: 101630af9729SIra Weiny cxl_map_device_regs(pdev, &cxlm->regs.device_regs, map); 101730af9729SIra Weiny dev_dbg(dev, "Probing device registers...\n"); 101830af9729SIra Weiny break; 101930af9729SIra Weiny default: 102030af9729SIra Weiny break; 102130af9729SIra Weiny } 102230af9729SIra Weiny 102330af9729SIra Weiny return 0; 102430af9729SIra Weiny } 102530af9729SIra Weiny 102607d62eacSIra Weiny static void cxl_decode_register_block(u32 reg_lo, u32 reg_hi, 102707d62eacSIra Weiny u8 *bar, u64 *offset, u8 *reg_type) 102807d62eacSIra Weiny { 102907d62eacSIra Weiny *offset = ((u64)reg_hi << 32) | (reg_lo & CXL_REGLOC_ADDR_MASK); 103007d62eacSIra Weiny *bar = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo); 103107d62eacSIra Weiny *reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); 103207d62eacSIra Weiny } 103307d62eacSIra Weiny 10341d5a4159SBen Widawsky /** 10351d5a4159SBen Widawsky * cxl_mem_setup_regs() - Setup necessary MMIO. 10361d5a4159SBen Widawsky * @cxlm: The CXL memory device to communicate with. 10371d5a4159SBen Widawsky * 10381d5a4159SBen Widawsky * Return: 0 if all necessary registers mapped. 10391d5a4159SBen Widawsky * 10401d5a4159SBen Widawsky * A memory device is required by spec to implement a certain set of MMIO 10411d5a4159SBen Widawsky * regions. The purpose of this function is to enumerate and map those 10421d5a4159SBen Widawsky * registers. 10431d5a4159SBen Widawsky */ 10441d5a4159SBen Widawsky static int cxl_mem_setup_regs(struct cxl_mem *cxlm) 10451d5a4159SBen Widawsky { 10461d5a4159SBen Widawsky struct pci_dev *pdev = cxlm->pdev; 10471d5a4159SBen Widawsky struct device *dev = &pdev->dev; 10481d5a4159SBen Widawsky u32 regloc_size, regblocks; 10496630d31cSBen Widawsky void __iomem *base; 10506630d31cSBen Widawsky int regloc, i; 105130af9729SIra Weiny struct cxl_register_map *map, *n; 105230af9729SIra Weiny LIST_HEAD(register_maps); 105330af9729SIra Weiny int ret = 0; 10541d5a4159SBen Widawsky 10551d5a4159SBen Widawsky regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_OFFSET); 10561d5a4159SBen Widawsky if (!regloc) { 10571d5a4159SBen Widawsky dev_err(dev, "register location dvsec not found\n"); 10581d5a4159SBen Widawsky return -ENXIO; 10591d5a4159SBen Widawsky } 10601d5a4159SBen Widawsky 1061f8a7e8c2SIra Weiny if (pci_request_mem_regions(pdev, pci_name(pdev))) 1062f8a7e8c2SIra Weiny return -ENODEV; 1063f8a7e8c2SIra Weiny 10641d5a4159SBen Widawsky /* Get the size of the Register Locator DVSEC */ 10651d5a4159SBen Widawsky pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size); 10661d5a4159SBen Widawsky regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size); 10671d5a4159SBen Widawsky 10681d5a4159SBen Widawsky regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET; 10691d5a4159SBen Widawsky regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8; 10701d5a4159SBen Widawsky 10711d5a4159SBen Widawsky for (i = 0; i < regblocks; i++, regloc += 8) { 10721d5a4159SBen Widawsky u32 reg_lo, reg_hi; 10731d5a4159SBen Widawsky u8 reg_type; 107407d62eacSIra Weiny u64 offset; 107507d62eacSIra Weiny u8 bar; 10761d5a4159SBen Widawsky 107730af9729SIra Weiny map = kzalloc(sizeof(*map), GFP_KERNEL); 107830af9729SIra Weiny if (!map) { 107930af9729SIra Weiny ret = -ENOMEM; 108030af9729SIra Weiny goto free_maps; 108130af9729SIra Weiny } 108230af9729SIra Weiny 108330af9729SIra Weiny list_add(&map->list, ®ister_maps); 108430af9729SIra Weiny 10851d5a4159SBen Widawsky pci_read_config_dword(pdev, regloc, ®_lo); 10861d5a4159SBen Widawsky pci_read_config_dword(pdev, regloc + 4, ®_hi); 10871d5a4159SBen Widawsky 108807d62eacSIra Weiny cxl_decode_register_block(reg_lo, reg_hi, &bar, &offset, 108907d62eacSIra Weiny ®_type); 109007d62eacSIra Weiny 109107d62eacSIra Weiny dev_dbg(dev, "Found register block in bar %u @ 0x%llx of type %u\n", 109207d62eacSIra Weiny bar, offset, reg_type); 10931d5a4159SBen Widawsky 109407d62eacSIra Weiny base = cxl_mem_map_regblock(cxlm, bar, offset); 109530af9729SIra Weiny if (!base) { 109630af9729SIra Weiny ret = -ENOMEM; 109730af9729SIra Weiny goto free_maps; 10981d5a4159SBen Widawsky } 10991d5a4159SBen Widawsky 110030af9729SIra Weiny map->barno = bar; 110130af9729SIra Weiny map->block_offset = offset; 110230af9729SIra Weiny map->reg_type = reg_type; 110330af9729SIra Weiny 110430af9729SIra Weiny ret = cxl_probe_regs(cxlm, base + offset, map); 110530af9729SIra Weiny 110630af9729SIra Weiny /* Always unmap the regblock regardless of probe success */ 110730af9729SIra Weiny cxl_mem_unmap_regblock(cxlm, base); 110830af9729SIra Weiny 110930af9729SIra Weiny if (ret) 111030af9729SIra Weiny goto free_maps; 11111d5a4159SBen Widawsky } 11121d5a4159SBen Widawsky 1113*9a016527SIra Weiny pci_release_mem_regions(pdev); 1114*9a016527SIra Weiny 111530af9729SIra Weiny list_for_each_entry(map, ®ister_maps, list) { 111630af9729SIra Weiny ret = cxl_map_regs(cxlm, map); 111730af9729SIra Weiny if (ret) 111830af9729SIra Weiny goto free_maps; 11191d5a4159SBen Widawsky } 11201d5a4159SBen Widawsky 112130af9729SIra Weiny free_maps: 112230af9729SIra Weiny list_for_each_entry_safe(map, n, ®ister_maps, list) { 112330af9729SIra Weiny list_del(&map->list); 112430af9729SIra Weiny kfree(map); 112530af9729SIra Weiny } 112630af9729SIra Weiny 112730af9729SIra Weiny return ret; 11281d5a4159SBen Widawsky } 11291d5a4159SBen Widawsky 113021e9f767SBen Widawsky static struct cxl_memdev *to_cxl_memdev(struct device *dev) 113121e9f767SBen Widawsky { 113221e9f767SBen Widawsky return container_of(dev, struct cxl_memdev, dev); 113321e9f767SBen Widawsky } 113421e9f767SBen Widawsky 113521e9f767SBen Widawsky static void cxl_memdev_release(struct device *dev) 113621e9f767SBen Widawsky { 113721e9f767SBen Widawsky struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 113821e9f767SBen Widawsky 113921e9f767SBen Widawsky ida_free(&cxl_memdev_ida, cxlmd->id); 114021e9f767SBen Widawsky kfree(cxlmd); 114121e9f767SBen Widawsky } 114221e9f767SBen Widawsky 114321e9f767SBen Widawsky static char *cxl_memdev_devnode(struct device *dev, umode_t *mode, kuid_t *uid, 114421e9f767SBen Widawsky kgid_t *gid) 114521e9f767SBen Widawsky { 114621e9f767SBen Widawsky return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); 114721e9f767SBen Widawsky } 114821e9f767SBen Widawsky 114921e9f767SBen Widawsky static ssize_t firmware_version_show(struct device *dev, 115021e9f767SBen Widawsky struct device_attribute *attr, char *buf) 115121e9f767SBen Widawsky { 115221e9f767SBen Widawsky struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 115321e9f767SBen Widawsky struct cxl_mem *cxlm = cxlmd->cxlm; 115421e9f767SBen Widawsky 115521e9f767SBen Widawsky return sysfs_emit(buf, "%.16s\n", cxlm->firmware_version); 115621e9f767SBen Widawsky } 115721e9f767SBen Widawsky static DEVICE_ATTR_RO(firmware_version); 115821e9f767SBen Widawsky 115921e9f767SBen Widawsky static ssize_t payload_max_show(struct device *dev, 116021e9f767SBen Widawsky struct device_attribute *attr, char *buf) 116121e9f767SBen Widawsky { 116221e9f767SBen Widawsky struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 116321e9f767SBen Widawsky struct cxl_mem *cxlm = cxlmd->cxlm; 116421e9f767SBen Widawsky 116521e9f767SBen Widawsky return sysfs_emit(buf, "%zu\n", cxlm->payload_size); 116621e9f767SBen Widawsky } 116721e9f767SBen Widawsky static DEVICE_ATTR_RO(payload_max); 116821e9f767SBen Widawsky 1169199cf8c3SVishal Verma static ssize_t label_storage_size_show(struct device *dev, 1170199cf8c3SVishal Verma struct device_attribute *attr, char *buf) 1171199cf8c3SVishal Verma { 1172199cf8c3SVishal Verma struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 1173199cf8c3SVishal Verma struct cxl_mem *cxlm = cxlmd->cxlm; 1174199cf8c3SVishal Verma 1175199cf8c3SVishal Verma return sysfs_emit(buf, "%zu\n", cxlm->lsa_size); 1176199cf8c3SVishal Verma } 1177199cf8c3SVishal Verma static DEVICE_ATTR_RO(label_storage_size); 1178199cf8c3SVishal Verma 117921e9f767SBen Widawsky static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, 118021e9f767SBen Widawsky char *buf) 118121e9f767SBen Widawsky { 118221e9f767SBen Widawsky struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 118321e9f767SBen Widawsky struct cxl_mem *cxlm = cxlmd->cxlm; 118421e9f767SBen Widawsky unsigned long long len = range_len(&cxlm->ram_range); 118521e9f767SBen Widawsky 118621e9f767SBen Widawsky return sysfs_emit(buf, "%#llx\n", len); 118721e9f767SBen Widawsky } 118821e9f767SBen Widawsky 118921e9f767SBen Widawsky static struct device_attribute dev_attr_ram_size = 119021e9f767SBen Widawsky __ATTR(size, 0444, ram_size_show, NULL); 119121e9f767SBen Widawsky 119221e9f767SBen Widawsky static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, 119321e9f767SBen Widawsky char *buf) 119421e9f767SBen Widawsky { 119521e9f767SBen Widawsky struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 119621e9f767SBen Widawsky struct cxl_mem *cxlm = cxlmd->cxlm; 119721e9f767SBen Widawsky unsigned long long len = range_len(&cxlm->pmem_range); 119821e9f767SBen Widawsky 119921e9f767SBen Widawsky return sysfs_emit(buf, "%#llx\n", len); 120021e9f767SBen Widawsky } 120121e9f767SBen Widawsky 120221e9f767SBen Widawsky static struct device_attribute dev_attr_pmem_size = 120321e9f767SBen Widawsky __ATTR(size, 0444, pmem_size_show, NULL); 120421e9f767SBen Widawsky 120521e9f767SBen Widawsky static struct attribute *cxl_memdev_attributes[] = { 120621e9f767SBen Widawsky &dev_attr_firmware_version.attr, 120721e9f767SBen Widawsky &dev_attr_payload_max.attr, 1208199cf8c3SVishal Verma &dev_attr_label_storage_size.attr, 120921e9f767SBen Widawsky NULL, 121021e9f767SBen Widawsky }; 121121e9f767SBen Widawsky 121221e9f767SBen Widawsky static struct attribute *cxl_memdev_pmem_attributes[] = { 121321e9f767SBen Widawsky &dev_attr_pmem_size.attr, 121421e9f767SBen Widawsky NULL, 121521e9f767SBen Widawsky }; 121621e9f767SBen Widawsky 121721e9f767SBen Widawsky static struct attribute *cxl_memdev_ram_attributes[] = { 121821e9f767SBen Widawsky &dev_attr_ram_size.attr, 121921e9f767SBen Widawsky NULL, 122021e9f767SBen Widawsky }; 122121e9f767SBen Widawsky 122221e9f767SBen Widawsky static struct attribute_group cxl_memdev_attribute_group = { 122321e9f767SBen Widawsky .attrs = cxl_memdev_attributes, 122421e9f767SBen Widawsky }; 122521e9f767SBen Widawsky 122621e9f767SBen Widawsky static struct attribute_group cxl_memdev_ram_attribute_group = { 122721e9f767SBen Widawsky .name = "ram", 122821e9f767SBen Widawsky .attrs = cxl_memdev_ram_attributes, 122921e9f767SBen Widawsky }; 123021e9f767SBen Widawsky 123121e9f767SBen Widawsky static struct attribute_group cxl_memdev_pmem_attribute_group = { 123221e9f767SBen Widawsky .name = "pmem", 123321e9f767SBen Widawsky .attrs = cxl_memdev_pmem_attributes, 123421e9f767SBen Widawsky }; 123521e9f767SBen Widawsky 123621e9f767SBen Widawsky static const struct attribute_group *cxl_memdev_attribute_groups[] = { 123721e9f767SBen Widawsky &cxl_memdev_attribute_group, 123821e9f767SBen Widawsky &cxl_memdev_ram_attribute_group, 123921e9f767SBen Widawsky &cxl_memdev_pmem_attribute_group, 124021e9f767SBen Widawsky NULL, 124121e9f767SBen Widawsky }; 124221e9f767SBen Widawsky 124321e9f767SBen Widawsky static const struct device_type cxl_memdev_type = { 124421e9f767SBen Widawsky .name = "cxl_memdev", 124521e9f767SBen Widawsky .release = cxl_memdev_release, 124621e9f767SBen Widawsky .devnode = cxl_memdev_devnode, 124721e9f767SBen Widawsky .groups = cxl_memdev_attribute_groups, 124821e9f767SBen Widawsky }; 124921e9f767SBen Widawsky 125021e9f767SBen Widawsky static void cxl_memdev_shutdown(struct cxl_memdev *cxlmd) 125121e9f767SBen Widawsky { 125221e9f767SBen Widawsky down_write(&cxl_memdev_rwsem); 125321e9f767SBen Widawsky cxlmd->cxlm = NULL; 125421e9f767SBen Widawsky up_write(&cxl_memdev_rwsem); 125521e9f767SBen Widawsky } 125621e9f767SBen Widawsky 125721e9f767SBen Widawsky static void cxl_memdev_unregister(void *_cxlmd) 125821e9f767SBen Widawsky { 125921e9f767SBen Widawsky struct cxl_memdev *cxlmd = _cxlmd; 126021e9f767SBen Widawsky struct device *dev = &cxlmd->dev; 126121e9f767SBen Widawsky 126221e9f767SBen Widawsky cdev_device_del(&cxlmd->cdev, dev); 126321e9f767SBen Widawsky cxl_memdev_shutdown(cxlmd); 126421e9f767SBen Widawsky put_device(dev); 126521e9f767SBen Widawsky } 126621e9f767SBen Widawsky 126721e9f767SBen Widawsky static struct cxl_memdev *cxl_memdev_alloc(struct cxl_mem *cxlm) 126821e9f767SBen Widawsky { 126921e9f767SBen Widawsky struct pci_dev *pdev = cxlm->pdev; 127021e9f767SBen Widawsky struct cxl_memdev *cxlmd; 127121e9f767SBen Widawsky struct device *dev; 127221e9f767SBen Widawsky struct cdev *cdev; 127321e9f767SBen Widawsky int rc; 127421e9f767SBen Widawsky 127521e9f767SBen Widawsky cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL); 127621e9f767SBen Widawsky if (!cxlmd) 127721e9f767SBen Widawsky return ERR_PTR(-ENOMEM); 127821e9f767SBen Widawsky 127921e9f767SBen Widawsky rc = ida_alloc_range(&cxl_memdev_ida, 0, CXL_MEM_MAX_DEVS, GFP_KERNEL); 128021e9f767SBen Widawsky if (rc < 0) 128121e9f767SBen Widawsky goto err; 128221e9f767SBen Widawsky cxlmd->id = rc; 128321e9f767SBen Widawsky 128421e9f767SBen Widawsky dev = &cxlmd->dev; 128521e9f767SBen Widawsky device_initialize(dev); 128621e9f767SBen Widawsky dev->parent = &pdev->dev; 128721e9f767SBen Widawsky dev->bus = &cxl_bus_type; 128821e9f767SBen Widawsky dev->devt = MKDEV(cxl_mem_major, cxlmd->id); 128921e9f767SBen Widawsky dev->type = &cxl_memdev_type; 129021e9f767SBen Widawsky device_set_pm_not_required(dev); 129121e9f767SBen Widawsky 129221e9f767SBen Widawsky cdev = &cxlmd->cdev; 129321e9f767SBen Widawsky cdev_init(cdev, &cxl_memdev_fops); 129421e9f767SBen Widawsky return cxlmd; 129521e9f767SBen Widawsky 129621e9f767SBen Widawsky err: 129721e9f767SBen Widawsky kfree(cxlmd); 129821e9f767SBen Widawsky return ERR_PTR(rc); 129921e9f767SBen Widawsky } 130021e9f767SBen Widawsky 130121e9f767SBen Widawsky static int cxl_mem_add_memdev(struct cxl_mem *cxlm) 130221e9f767SBen Widawsky { 130321e9f767SBen Widawsky struct cxl_memdev *cxlmd; 130421e9f767SBen Widawsky struct device *dev; 130521e9f767SBen Widawsky struct cdev *cdev; 130621e9f767SBen Widawsky int rc; 130721e9f767SBen Widawsky 130821e9f767SBen Widawsky cxlmd = cxl_memdev_alloc(cxlm); 130921e9f767SBen Widawsky if (IS_ERR(cxlmd)) 131021e9f767SBen Widawsky return PTR_ERR(cxlmd); 131121e9f767SBen Widawsky 131221e9f767SBen Widawsky dev = &cxlmd->dev; 131321e9f767SBen Widawsky rc = dev_set_name(dev, "mem%d", cxlmd->id); 131421e9f767SBen Widawsky if (rc) 131521e9f767SBen Widawsky goto err; 131621e9f767SBen Widawsky 131721e9f767SBen Widawsky /* 131821e9f767SBen Widawsky * Activate ioctl operations, no cxl_memdev_rwsem manipulation 131921e9f767SBen Widawsky * needed as this is ordered with cdev_add() publishing the device. 132021e9f767SBen Widawsky */ 132121e9f767SBen Widawsky cxlmd->cxlm = cxlm; 132221e9f767SBen Widawsky 132321e9f767SBen Widawsky cdev = &cxlmd->cdev; 132421e9f767SBen Widawsky rc = cdev_device_add(cdev, dev); 132521e9f767SBen Widawsky if (rc) 132621e9f767SBen Widawsky goto err; 132721e9f767SBen Widawsky 132821e9f767SBen Widawsky return devm_add_action_or_reset(dev->parent, cxl_memdev_unregister, 132921e9f767SBen Widawsky cxlmd); 133021e9f767SBen Widawsky 133121e9f767SBen Widawsky err: 133221e9f767SBen Widawsky /* 133321e9f767SBen Widawsky * The cdev was briefly live, shutdown any ioctl operations that 133421e9f767SBen Widawsky * saw that state. 133521e9f767SBen Widawsky */ 133621e9f767SBen Widawsky cxl_memdev_shutdown(cxlmd); 133721e9f767SBen Widawsky put_device(dev); 133821e9f767SBen Widawsky return rc; 133921e9f767SBen Widawsky } 134021e9f767SBen Widawsky 134121e9f767SBen Widawsky static int cxl_xfer_log(struct cxl_mem *cxlm, uuid_t *uuid, u32 size, u8 *out) 134221e9f767SBen Widawsky { 134321e9f767SBen Widawsky u32 remaining = size; 134421e9f767SBen Widawsky u32 offset = 0; 134521e9f767SBen Widawsky 134621e9f767SBen Widawsky while (remaining) { 134721e9f767SBen Widawsky u32 xfer_size = min_t(u32, remaining, cxlm->payload_size); 134821e9f767SBen Widawsky struct cxl_mbox_get_log { 134921e9f767SBen Widawsky uuid_t uuid; 135021e9f767SBen Widawsky __le32 offset; 135121e9f767SBen Widawsky __le32 length; 135221e9f767SBen Widawsky } __packed log = { 135321e9f767SBen Widawsky .uuid = *uuid, 135421e9f767SBen Widawsky .offset = cpu_to_le32(offset), 135521e9f767SBen Widawsky .length = cpu_to_le32(xfer_size) 135621e9f767SBen Widawsky }; 135721e9f767SBen Widawsky int rc; 135821e9f767SBen Widawsky 135921e9f767SBen Widawsky rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_LOG, &log, 136021e9f767SBen Widawsky sizeof(log), out, xfer_size); 136121e9f767SBen Widawsky if (rc < 0) 136221e9f767SBen Widawsky return rc; 136321e9f767SBen Widawsky 136421e9f767SBen Widawsky out += xfer_size; 136521e9f767SBen Widawsky remaining -= xfer_size; 136621e9f767SBen Widawsky offset += xfer_size; 136721e9f767SBen Widawsky } 136821e9f767SBen Widawsky 136921e9f767SBen Widawsky return 0; 137021e9f767SBen Widawsky } 137121e9f767SBen Widawsky 137221e9f767SBen Widawsky /** 137321e9f767SBen Widawsky * cxl_walk_cel() - Walk through the Command Effects Log. 137421e9f767SBen Widawsky * @cxlm: Device. 137521e9f767SBen Widawsky * @size: Length of the Command Effects Log. 137621e9f767SBen Widawsky * @cel: CEL 137721e9f767SBen Widawsky * 137821e9f767SBen Widawsky * Iterate over each entry in the CEL and determine if the driver supports the 137921e9f767SBen Widawsky * command. If so, the command is enabled for the device and can be used later. 138021e9f767SBen Widawsky */ 138121e9f767SBen Widawsky static void cxl_walk_cel(struct cxl_mem *cxlm, size_t size, u8 *cel) 138221e9f767SBen Widawsky { 138321e9f767SBen Widawsky struct cel_entry { 138421e9f767SBen Widawsky __le16 opcode; 138521e9f767SBen Widawsky __le16 effect; 138621e9f767SBen Widawsky } __packed * cel_entry; 138721e9f767SBen Widawsky const int cel_entries = size / sizeof(*cel_entry); 138821e9f767SBen Widawsky int i; 138921e9f767SBen Widawsky 139021e9f767SBen Widawsky cel_entry = (struct cel_entry *)cel; 139121e9f767SBen Widawsky 139221e9f767SBen Widawsky for (i = 0; i < cel_entries; i++) { 139321e9f767SBen Widawsky u16 opcode = le16_to_cpu(cel_entry[i].opcode); 139421e9f767SBen Widawsky struct cxl_mem_command *cmd = cxl_mem_find_command(opcode); 139521e9f767SBen Widawsky 139621e9f767SBen Widawsky if (!cmd) { 139721e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, 139821e9f767SBen Widawsky "Opcode 0x%04x unsupported by driver", opcode); 139921e9f767SBen Widawsky continue; 140021e9f767SBen Widawsky } 140121e9f767SBen Widawsky 140221e9f767SBen Widawsky set_bit(cmd->info.id, cxlm->enabled_cmds); 140321e9f767SBen Widawsky } 140421e9f767SBen Widawsky } 140521e9f767SBen Widawsky 140621e9f767SBen Widawsky struct cxl_mbox_get_supported_logs { 140721e9f767SBen Widawsky __le16 entries; 140821e9f767SBen Widawsky u8 rsvd[6]; 140921e9f767SBen Widawsky struct gsl_entry { 141021e9f767SBen Widawsky uuid_t uuid; 141121e9f767SBen Widawsky __le32 size; 141221e9f767SBen Widawsky } __packed entry[]; 141321e9f767SBen Widawsky } __packed; 141421e9f767SBen Widawsky 141521e9f767SBen Widawsky static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_mem *cxlm) 141621e9f767SBen Widawsky { 141721e9f767SBen Widawsky struct cxl_mbox_get_supported_logs *ret; 141821e9f767SBen Widawsky int rc; 141921e9f767SBen Widawsky 142021e9f767SBen Widawsky ret = kvmalloc(cxlm->payload_size, GFP_KERNEL); 142121e9f767SBen Widawsky if (!ret) 142221e9f767SBen Widawsky return ERR_PTR(-ENOMEM); 142321e9f767SBen Widawsky 142421e9f767SBen Widawsky rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_SUPPORTED_LOGS, NULL, 142521e9f767SBen Widawsky 0, ret, cxlm->payload_size); 142621e9f767SBen Widawsky if (rc < 0) { 142721e9f767SBen Widawsky kvfree(ret); 142821e9f767SBen Widawsky return ERR_PTR(rc); 142921e9f767SBen Widawsky } 143021e9f767SBen Widawsky 143121e9f767SBen Widawsky return ret; 143221e9f767SBen Widawsky } 143321e9f767SBen Widawsky 143421e9f767SBen Widawsky /** 143521e9f767SBen Widawsky * cxl_mem_enumerate_cmds() - Enumerate commands for a device. 143621e9f767SBen Widawsky * @cxlm: The device. 143721e9f767SBen Widawsky * 143821e9f767SBen Widawsky * Returns 0 if enumerate completed successfully. 143921e9f767SBen Widawsky * 144021e9f767SBen Widawsky * CXL devices have optional support for certain commands. This function will 144121e9f767SBen Widawsky * determine the set of supported commands for the hardware and update the 144221e9f767SBen Widawsky * enabled_cmds bitmap in the @cxlm. 144321e9f767SBen Widawsky */ 144421e9f767SBen Widawsky static int cxl_mem_enumerate_cmds(struct cxl_mem *cxlm) 144521e9f767SBen Widawsky { 144621e9f767SBen Widawsky struct cxl_mbox_get_supported_logs *gsl; 144721e9f767SBen Widawsky struct device *dev = &cxlm->pdev->dev; 144821e9f767SBen Widawsky struct cxl_mem_command *cmd; 144921e9f767SBen Widawsky int i, rc; 145021e9f767SBen Widawsky 145121e9f767SBen Widawsky gsl = cxl_get_gsl(cxlm); 145221e9f767SBen Widawsky if (IS_ERR(gsl)) 145321e9f767SBen Widawsky return PTR_ERR(gsl); 145421e9f767SBen Widawsky 145521e9f767SBen Widawsky rc = -ENOENT; 145621e9f767SBen Widawsky for (i = 0; i < le16_to_cpu(gsl->entries); i++) { 145721e9f767SBen Widawsky u32 size = le32_to_cpu(gsl->entry[i].size); 145821e9f767SBen Widawsky uuid_t uuid = gsl->entry[i].uuid; 145921e9f767SBen Widawsky u8 *log; 146021e9f767SBen Widawsky 146121e9f767SBen Widawsky dev_dbg(dev, "Found LOG type %pU of size %d", &uuid, size); 146221e9f767SBen Widawsky 146321e9f767SBen Widawsky if (!uuid_equal(&uuid, &log_uuid[CEL_UUID])) 146421e9f767SBen Widawsky continue; 146521e9f767SBen Widawsky 146621e9f767SBen Widawsky log = kvmalloc(size, GFP_KERNEL); 146721e9f767SBen Widawsky if (!log) { 146821e9f767SBen Widawsky rc = -ENOMEM; 146921e9f767SBen Widawsky goto out; 147021e9f767SBen Widawsky } 147121e9f767SBen Widawsky 147221e9f767SBen Widawsky rc = cxl_xfer_log(cxlm, &uuid, size, log); 147321e9f767SBen Widawsky if (rc) { 147421e9f767SBen Widawsky kvfree(log); 147521e9f767SBen Widawsky goto out; 147621e9f767SBen Widawsky } 147721e9f767SBen Widawsky 147821e9f767SBen Widawsky cxl_walk_cel(cxlm, size, log); 147921e9f767SBen Widawsky kvfree(log); 148021e9f767SBen Widawsky 148121e9f767SBen Widawsky /* In case CEL was bogus, enable some default commands. */ 148221e9f767SBen Widawsky cxl_for_each_cmd(cmd) 148321e9f767SBen Widawsky if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE) 148421e9f767SBen Widawsky set_bit(cmd->info.id, cxlm->enabled_cmds); 148521e9f767SBen Widawsky 148621e9f767SBen Widawsky /* Found the required CEL */ 148721e9f767SBen Widawsky rc = 0; 148821e9f767SBen Widawsky } 148921e9f767SBen Widawsky 149021e9f767SBen Widawsky out: 149121e9f767SBen Widawsky kvfree(gsl); 149221e9f767SBen Widawsky return rc; 149321e9f767SBen Widawsky } 149421e9f767SBen Widawsky 149521e9f767SBen Widawsky /** 149621e9f767SBen Widawsky * cxl_mem_identify() - Send the IDENTIFY command to the device. 149721e9f767SBen Widawsky * @cxlm: The device to identify. 149821e9f767SBen Widawsky * 149921e9f767SBen Widawsky * Return: 0 if identify was executed successfully. 150021e9f767SBen Widawsky * 150121e9f767SBen Widawsky * This will dispatch the identify command to the device and on success populate 150221e9f767SBen Widawsky * structures to be exported to sysfs. 150321e9f767SBen Widawsky */ 150421e9f767SBen Widawsky static int cxl_mem_identify(struct cxl_mem *cxlm) 150521e9f767SBen Widawsky { 150621e9f767SBen Widawsky /* See CXL 2.0 Table 175 Identify Memory Device Output Payload */ 150721e9f767SBen Widawsky struct cxl_mbox_identify { 150821e9f767SBen Widawsky char fw_revision[0x10]; 150921e9f767SBen Widawsky __le64 total_capacity; 151021e9f767SBen Widawsky __le64 volatile_capacity; 151121e9f767SBen Widawsky __le64 persistent_capacity; 151221e9f767SBen Widawsky __le64 partition_align; 151321e9f767SBen Widawsky __le16 info_event_log_size; 151421e9f767SBen Widawsky __le16 warning_event_log_size; 151521e9f767SBen Widawsky __le16 failure_event_log_size; 151621e9f767SBen Widawsky __le16 fatal_event_log_size; 151721e9f767SBen Widawsky __le32 lsa_size; 151821e9f767SBen Widawsky u8 poison_list_max_mer[3]; 151921e9f767SBen Widawsky __le16 inject_poison_limit; 152021e9f767SBen Widawsky u8 poison_caps; 152121e9f767SBen Widawsky u8 qos_telemetry_caps; 152221e9f767SBen Widawsky } __packed id; 152321e9f767SBen Widawsky int rc; 152421e9f767SBen Widawsky 152521e9f767SBen Widawsky rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_IDENTIFY, NULL, 0, &id, 152621e9f767SBen Widawsky sizeof(id)); 152721e9f767SBen Widawsky if (rc < 0) 152821e9f767SBen Widawsky return rc; 152921e9f767SBen Widawsky 153021e9f767SBen Widawsky /* 153121e9f767SBen Widawsky * TODO: enumerate DPA map, as 'ram' and 'pmem' do not alias. 153221e9f767SBen Widawsky * For now, only the capacity is exported in sysfs 153321e9f767SBen Widawsky */ 153421e9f767SBen Widawsky cxlm->ram_range.start = 0; 153521e9f767SBen Widawsky cxlm->ram_range.end = le64_to_cpu(id.volatile_capacity) * SZ_256M - 1; 153621e9f767SBen Widawsky 153721e9f767SBen Widawsky cxlm->pmem_range.start = 0; 153821e9f767SBen Widawsky cxlm->pmem_range.end = 153921e9f767SBen Widawsky le64_to_cpu(id.persistent_capacity) * SZ_256M - 1; 154021e9f767SBen Widawsky 1541199cf8c3SVishal Verma cxlm->lsa_size = le32_to_cpu(id.lsa_size); 154221e9f767SBen Widawsky memcpy(cxlm->firmware_version, id.fw_revision, sizeof(id.fw_revision)); 154321e9f767SBen Widawsky 154421e9f767SBen Widawsky return 0; 154521e9f767SBen Widawsky } 154621e9f767SBen Widawsky 154721e9f767SBen Widawsky static int cxl_mem_probe(struct pci_dev *pdev, const struct pci_device_id *id) 154821e9f767SBen Widawsky { 15491b0a1a2aSBen Widawsky struct cxl_mem *cxlm; 15501d5a4159SBen Widawsky int rc; 155121e9f767SBen Widawsky 155221e9f767SBen Widawsky rc = pcim_enable_device(pdev); 155321e9f767SBen Widawsky if (rc) 155421e9f767SBen Widawsky return rc; 155521e9f767SBen Widawsky 15561b0a1a2aSBen Widawsky cxlm = cxl_mem_create(pdev); 15571b0a1a2aSBen Widawsky if (IS_ERR(cxlm)) 15581b0a1a2aSBen Widawsky return PTR_ERR(cxlm); 15591b0a1a2aSBen Widawsky 156021e9f767SBen Widawsky rc = cxl_mem_setup_regs(cxlm); 156121e9f767SBen Widawsky if (rc) 156221e9f767SBen Widawsky return rc; 156321e9f767SBen Widawsky 156421e9f767SBen Widawsky rc = cxl_mem_setup_mailbox(cxlm); 156521e9f767SBen Widawsky if (rc) 156621e9f767SBen Widawsky return rc; 156721e9f767SBen Widawsky 156821e9f767SBen Widawsky rc = cxl_mem_enumerate_cmds(cxlm); 156921e9f767SBen Widawsky if (rc) 157021e9f767SBen Widawsky return rc; 157121e9f767SBen Widawsky 157221e9f767SBen Widawsky rc = cxl_mem_identify(cxlm); 157321e9f767SBen Widawsky if (rc) 157421e9f767SBen Widawsky return rc; 157521e9f767SBen Widawsky 157621e9f767SBen Widawsky return cxl_mem_add_memdev(cxlm); 157721e9f767SBen Widawsky } 157821e9f767SBen Widawsky 157921e9f767SBen Widawsky static const struct pci_device_id cxl_mem_pci_tbl[] = { 158021e9f767SBen Widawsky /* PCI class code for CXL.mem Type-3 Devices */ 158121e9f767SBen Widawsky { PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)}, 158221e9f767SBen Widawsky { /* terminate list */ }, 158321e9f767SBen Widawsky }; 158421e9f767SBen Widawsky MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl); 158521e9f767SBen Widawsky 158621e9f767SBen Widawsky static struct pci_driver cxl_mem_driver = { 158721e9f767SBen Widawsky .name = KBUILD_MODNAME, 158821e9f767SBen Widawsky .id_table = cxl_mem_pci_tbl, 158921e9f767SBen Widawsky .probe = cxl_mem_probe, 159021e9f767SBen Widawsky .driver = { 159121e9f767SBen Widawsky .probe_type = PROBE_PREFER_ASYNCHRONOUS, 159221e9f767SBen Widawsky }, 159321e9f767SBen Widawsky }; 159421e9f767SBen Widawsky 159521e9f767SBen Widawsky static __init int cxl_mem_init(void) 159621e9f767SBen Widawsky { 159721e9f767SBen Widawsky struct dentry *mbox_debugfs; 159821e9f767SBen Widawsky dev_t devt; 159921e9f767SBen Widawsky int rc; 160021e9f767SBen Widawsky 160121e9f767SBen Widawsky /* Double check the anonymous union trickery in struct cxl_regs */ 160221e9f767SBen Widawsky BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) != 160321e9f767SBen Widawsky offsetof(struct cxl_regs, device_regs.memdev)); 160421e9f767SBen Widawsky 160521e9f767SBen Widawsky rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); 160621e9f767SBen Widawsky if (rc) 160721e9f767SBen Widawsky return rc; 160821e9f767SBen Widawsky 160921e9f767SBen Widawsky cxl_mem_major = MAJOR(devt); 161021e9f767SBen Widawsky 161121e9f767SBen Widawsky rc = pci_register_driver(&cxl_mem_driver); 161221e9f767SBen Widawsky if (rc) { 161321e9f767SBen Widawsky unregister_chrdev_region(MKDEV(cxl_mem_major, 0), 161421e9f767SBen Widawsky CXL_MEM_MAX_DEVS); 161521e9f767SBen Widawsky return rc; 161621e9f767SBen Widawsky } 161721e9f767SBen Widawsky 161821e9f767SBen Widawsky cxl_debugfs = debugfs_create_dir("cxl", NULL); 161921e9f767SBen Widawsky mbox_debugfs = debugfs_create_dir("mbox", cxl_debugfs); 162021e9f767SBen Widawsky debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs, 162121e9f767SBen Widawsky &cxl_raw_allow_all); 162221e9f767SBen Widawsky 162321e9f767SBen Widawsky return 0; 162421e9f767SBen Widawsky } 162521e9f767SBen Widawsky 162621e9f767SBen Widawsky static __exit void cxl_mem_exit(void) 162721e9f767SBen Widawsky { 162821e9f767SBen Widawsky debugfs_remove_recursive(cxl_debugfs); 162921e9f767SBen Widawsky pci_unregister_driver(&cxl_mem_driver); 163021e9f767SBen Widawsky unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS); 163121e9f767SBen Widawsky } 163221e9f767SBen Widawsky 163321e9f767SBen Widawsky MODULE_LICENSE("GPL v2"); 163421e9f767SBen Widawsky module_init(cxl_mem_init); 163521e9f767SBen Widawsky module_exit(cxl_mem_exit); 163621e9f767SBen Widawsky MODULE_IMPORT_NS(CXL); 1637