121e9f767SBen Widawsky // SPDX-License-Identifier: GPL-2.0-only 221e9f767SBen Widawsky /* Copyright(c) 2020 Intel Corporation. All rights reserved. */ 321e9f767SBen Widawsky #include <uapi/linux/cxl_mem.h> 421e9f767SBen Widawsky #include <linux/security.h> 521e9f767SBen Widawsky #include <linux/debugfs.h> 621e9f767SBen Widawsky #include <linux/module.h> 721e9f767SBen Widawsky #include <linux/sizes.h> 821e9f767SBen Widawsky #include <linux/mutex.h> 921e9f767SBen Widawsky #include <linux/cdev.h> 1021e9f767SBen Widawsky #include <linux/idr.h> 1121e9f767SBen Widawsky #include <linux/pci.h> 1221e9f767SBen Widawsky #include <linux/io.h> 1321e9f767SBen Widawsky #include <linux/io-64-nonatomic-lo-hi.h> 1421e9f767SBen Widawsky #include "pci.h" 1521e9f767SBen Widawsky #include "cxl.h" 1621e9f767SBen Widawsky #include "mem.h" 1721e9f767SBen Widawsky 1821e9f767SBen Widawsky /** 1921e9f767SBen Widawsky * DOC: cxl pci 2021e9f767SBen Widawsky * 2121e9f767SBen Widawsky * This implements the PCI exclusive functionality for a CXL device as it is 2221e9f767SBen Widawsky * defined by the Compute Express Link specification. CXL devices may surface 2321e9f767SBen Widawsky * certain functionality even if it isn't CXL enabled. 2421e9f767SBen Widawsky * 2521e9f767SBen Widawsky * The driver has several responsibilities, mainly: 2621e9f767SBen Widawsky * - Create the memX device and register on the CXL bus. 2721e9f767SBen Widawsky * - Enumerate device's register interface and map them. 2821e9f767SBen Widawsky * - Probe the device attributes to establish sysfs interface. 2921e9f767SBen Widawsky * - Provide an IOCTL interface to userspace to communicate with the device for 3021e9f767SBen Widawsky * things like firmware update. 3121e9f767SBen Widawsky */ 3221e9f767SBen Widawsky 3321e9f767SBen Widawsky #define cxl_doorbell_busy(cxlm) \ 3421e9f767SBen Widawsky (readl((cxlm)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \ 3521e9f767SBen Widawsky CXLDEV_MBOX_CTRL_DOORBELL) 3621e9f767SBen Widawsky 3721e9f767SBen Widawsky /* CXL 2.0 - 8.2.8.4 */ 3821e9f767SBen Widawsky #define CXL_MAILBOX_TIMEOUT_MS (2 * HZ) 3921e9f767SBen Widawsky 4021e9f767SBen Widawsky enum opcode { 4121e9f767SBen Widawsky CXL_MBOX_OP_INVALID = 0x0000, 4221e9f767SBen Widawsky CXL_MBOX_OP_RAW = CXL_MBOX_OP_INVALID, 4321e9f767SBen Widawsky CXL_MBOX_OP_GET_FW_INFO = 0x0200, 4421e9f767SBen Widawsky CXL_MBOX_OP_ACTIVATE_FW = 0x0202, 4521e9f767SBen Widawsky CXL_MBOX_OP_GET_SUPPORTED_LOGS = 0x0400, 4621e9f767SBen Widawsky CXL_MBOX_OP_GET_LOG = 0x0401, 4721e9f767SBen Widawsky CXL_MBOX_OP_IDENTIFY = 0x4000, 4821e9f767SBen Widawsky CXL_MBOX_OP_GET_PARTITION_INFO = 0x4100, 4921e9f767SBen Widawsky CXL_MBOX_OP_SET_PARTITION_INFO = 0x4101, 5021e9f767SBen Widawsky CXL_MBOX_OP_GET_LSA = 0x4102, 5121e9f767SBen Widawsky CXL_MBOX_OP_SET_LSA = 0x4103, 5221e9f767SBen Widawsky CXL_MBOX_OP_GET_HEALTH_INFO = 0x4200, 5321e9f767SBen Widawsky CXL_MBOX_OP_SET_SHUTDOWN_STATE = 0x4204, 5421e9f767SBen Widawsky CXL_MBOX_OP_SCAN_MEDIA = 0x4304, 5521e9f767SBen Widawsky CXL_MBOX_OP_GET_SCAN_MEDIA = 0x4305, 5621e9f767SBen Widawsky CXL_MBOX_OP_MAX = 0x10000 5721e9f767SBen Widawsky }; 5821e9f767SBen Widawsky 5921e9f767SBen Widawsky /** 6021e9f767SBen Widawsky * struct mbox_cmd - A command to be submitted to hardware. 6121e9f767SBen Widawsky * @opcode: (input) The command set and command submitted to hardware. 6221e9f767SBen Widawsky * @payload_in: (input) Pointer to the input payload. 6321e9f767SBen Widawsky * @payload_out: (output) Pointer to the output payload. Must be allocated by 6421e9f767SBen Widawsky * the caller. 6521e9f767SBen Widawsky * @size_in: (input) Number of bytes to load from @payload_in. 6621e9f767SBen Widawsky * @size_out: (input) Max number of bytes loaded into @payload_out. 6721e9f767SBen Widawsky * (output) Number of bytes generated by the device. For fixed size 6821e9f767SBen Widawsky * outputs commands this is always expected to be deterministic. For 6921e9f767SBen Widawsky * variable sized output commands, it tells the exact number of bytes 7021e9f767SBen Widawsky * written. 7121e9f767SBen Widawsky * @return_code: (output) Error code returned from hardware. 7221e9f767SBen Widawsky * 7321e9f767SBen Widawsky * This is the primary mechanism used to send commands to the hardware. 7421e9f767SBen Widawsky * All the fields except @payload_* correspond exactly to the fields described in 7521e9f767SBen Widawsky * Command Register section of the CXL 2.0 8.2.8.4.5. @payload_in and 7621e9f767SBen Widawsky * @payload_out are written to, and read from the Command Payload Registers 7721e9f767SBen Widawsky * defined in CXL 2.0 8.2.8.4.8. 7821e9f767SBen Widawsky */ 7921e9f767SBen Widawsky struct mbox_cmd { 8021e9f767SBen Widawsky u16 opcode; 8121e9f767SBen Widawsky void *payload_in; 8221e9f767SBen Widawsky void *payload_out; 8321e9f767SBen Widawsky size_t size_in; 8421e9f767SBen Widawsky size_t size_out; 8521e9f767SBen Widawsky u16 return_code; 8621e9f767SBen Widawsky #define CXL_MBOX_SUCCESS 0 8721e9f767SBen Widawsky }; 8821e9f767SBen Widawsky 8921e9f767SBen Widawsky static int cxl_mem_major; 9021e9f767SBen Widawsky static DEFINE_IDA(cxl_memdev_ida); 9121e9f767SBen Widawsky static DECLARE_RWSEM(cxl_memdev_rwsem); 9221e9f767SBen Widawsky static struct dentry *cxl_debugfs; 9321e9f767SBen Widawsky static bool cxl_raw_allow_all; 9421e9f767SBen Widawsky 9521e9f767SBen Widawsky enum { 9621e9f767SBen Widawsky CEL_UUID, 9721e9f767SBen Widawsky VENDOR_DEBUG_UUID, 9821e9f767SBen Widawsky }; 9921e9f767SBen Widawsky 10021e9f767SBen Widawsky /* See CXL 2.0 Table 170. Get Log Input Payload */ 10121e9f767SBen Widawsky static const uuid_t log_uuid[] = { 10221e9f767SBen Widawsky [CEL_UUID] = UUID_INIT(0xda9c0b5, 0xbf41, 0x4b78, 0x8f, 0x79, 0x96, 10321e9f767SBen Widawsky 0xb1, 0x62, 0x3b, 0x3f, 0x17), 10421e9f767SBen Widawsky [VENDOR_DEBUG_UUID] = UUID_INIT(0xe1819d9, 0x11a9, 0x400c, 0x81, 0x1f, 10521e9f767SBen Widawsky 0xd6, 0x07, 0x19, 0x40, 0x3d, 0x86), 10621e9f767SBen Widawsky }; 10721e9f767SBen Widawsky 10821e9f767SBen Widawsky /** 10921e9f767SBen Widawsky * struct cxl_mem_command - Driver representation of a memory device command 11021e9f767SBen Widawsky * @info: Command information as it exists for the UAPI 11121e9f767SBen Widawsky * @opcode: The actual bits used for the mailbox protocol 11221e9f767SBen Widawsky * @flags: Set of flags effecting driver behavior. 11321e9f767SBen Widawsky * 11421e9f767SBen Widawsky * * %CXL_CMD_FLAG_FORCE_ENABLE: In cases of error, commands with this flag 11521e9f767SBen Widawsky * will be enabled by the driver regardless of what hardware may have 11621e9f767SBen Widawsky * advertised. 11721e9f767SBen Widawsky * 11821e9f767SBen Widawsky * The cxl_mem_command is the driver's internal representation of commands that 11921e9f767SBen Widawsky * are supported by the driver. Some of these commands may not be supported by 12021e9f767SBen Widawsky * the hardware. The driver will use @info to validate the fields passed in by 12121e9f767SBen Widawsky * the user then submit the @opcode to the hardware. 12221e9f767SBen Widawsky * 12321e9f767SBen Widawsky * See struct cxl_command_info. 12421e9f767SBen Widawsky */ 12521e9f767SBen Widawsky struct cxl_mem_command { 12621e9f767SBen Widawsky struct cxl_command_info info; 12721e9f767SBen Widawsky enum opcode opcode; 12821e9f767SBen Widawsky u32 flags; 12921e9f767SBen Widawsky #define CXL_CMD_FLAG_NONE 0 13021e9f767SBen Widawsky #define CXL_CMD_FLAG_FORCE_ENABLE BIT(0) 13121e9f767SBen Widawsky }; 13221e9f767SBen Widawsky 13321e9f767SBen Widawsky #define CXL_CMD(_id, sin, sout, _flags) \ 13421e9f767SBen Widawsky [CXL_MEM_COMMAND_ID_##_id] = { \ 13521e9f767SBen Widawsky .info = { \ 13621e9f767SBen Widawsky .id = CXL_MEM_COMMAND_ID_##_id, \ 13721e9f767SBen Widawsky .size_in = sin, \ 13821e9f767SBen Widawsky .size_out = sout, \ 13921e9f767SBen Widawsky }, \ 14021e9f767SBen Widawsky .opcode = CXL_MBOX_OP_##_id, \ 14121e9f767SBen Widawsky .flags = _flags, \ 14221e9f767SBen Widawsky } 14321e9f767SBen Widawsky 14421e9f767SBen Widawsky /* 14521e9f767SBen Widawsky * This table defines the supported mailbox commands for the driver. This table 14621e9f767SBen Widawsky * is made up of a UAPI structure. Non-negative values as parameters in the 14721e9f767SBen Widawsky * table will be validated against the user's input. For example, if size_in is 14821e9f767SBen Widawsky * 0, and the user passed in 1, it is an error. 14921e9f767SBen Widawsky */ 15021e9f767SBen Widawsky static struct cxl_mem_command mem_commands[CXL_MEM_COMMAND_ID_MAX] = { 15121e9f767SBen Widawsky CXL_CMD(IDENTIFY, 0, 0x43, CXL_CMD_FLAG_FORCE_ENABLE), 15221e9f767SBen Widawsky #ifdef CONFIG_CXL_MEM_RAW_COMMANDS 15321e9f767SBen Widawsky CXL_CMD(RAW, ~0, ~0, 0), 15421e9f767SBen Widawsky #endif 15521e9f767SBen Widawsky CXL_CMD(GET_SUPPORTED_LOGS, 0, ~0, CXL_CMD_FLAG_FORCE_ENABLE), 15621e9f767SBen Widawsky CXL_CMD(GET_FW_INFO, 0, 0x50, 0), 15721e9f767SBen Widawsky CXL_CMD(GET_PARTITION_INFO, 0, 0x20, 0), 15821e9f767SBen Widawsky CXL_CMD(GET_LSA, 0x8, ~0, 0), 15921e9f767SBen Widawsky CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0), 16021e9f767SBen Widawsky CXL_CMD(GET_LOG, 0x18, ~0, CXL_CMD_FLAG_FORCE_ENABLE), 16121e9f767SBen Widawsky }; 16221e9f767SBen Widawsky 16321e9f767SBen Widawsky /* 16421e9f767SBen Widawsky * Commands that RAW doesn't permit. The rationale for each: 16521e9f767SBen Widawsky * 16621e9f767SBen Widawsky * CXL_MBOX_OP_ACTIVATE_FW: Firmware activation requires adjustment / 16721e9f767SBen Widawsky * coordination of transaction timeout values at the root bridge level. 16821e9f767SBen Widawsky * 16921e9f767SBen Widawsky * CXL_MBOX_OP_SET_PARTITION_INFO: The device memory map may change live 17021e9f767SBen Widawsky * and needs to be coordinated with HDM updates. 17121e9f767SBen Widawsky * 17221e9f767SBen Widawsky * CXL_MBOX_OP_SET_LSA: The label storage area may be cached by the 17321e9f767SBen Widawsky * driver and any writes from userspace invalidates those contents. 17421e9f767SBen Widawsky * 17521e9f767SBen Widawsky * CXL_MBOX_OP_SET_SHUTDOWN_STATE: Set shutdown state assumes no writes 17621e9f767SBen Widawsky * to the device after it is marked clean, userspace can not make that 17721e9f767SBen Widawsky * assertion. 17821e9f767SBen Widawsky * 17921e9f767SBen Widawsky * CXL_MBOX_OP_[GET_]SCAN_MEDIA: The kernel provides a native error list that 18021e9f767SBen Widawsky * is kept up to date with patrol notifications and error management. 18121e9f767SBen Widawsky */ 18221e9f767SBen Widawsky static u16 cxl_disabled_raw_commands[] = { 18321e9f767SBen Widawsky CXL_MBOX_OP_ACTIVATE_FW, 18421e9f767SBen Widawsky CXL_MBOX_OP_SET_PARTITION_INFO, 18521e9f767SBen Widawsky CXL_MBOX_OP_SET_LSA, 18621e9f767SBen Widawsky CXL_MBOX_OP_SET_SHUTDOWN_STATE, 18721e9f767SBen Widawsky CXL_MBOX_OP_SCAN_MEDIA, 18821e9f767SBen Widawsky CXL_MBOX_OP_GET_SCAN_MEDIA, 18921e9f767SBen Widawsky }; 19021e9f767SBen Widawsky 19121e9f767SBen Widawsky /* 19221e9f767SBen Widawsky * Command sets that RAW doesn't permit. All opcodes in this set are 19321e9f767SBen Widawsky * disabled because they pass plain text security payloads over the 19421e9f767SBen Widawsky * user/kernel boundary. This functionality is intended to be wrapped 19521e9f767SBen Widawsky * behind the keys ABI which allows for encrypted payloads in the UAPI 19621e9f767SBen Widawsky */ 19721e9f767SBen Widawsky static u8 security_command_sets[] = { 19821e9f767SBen Widawsky 0x44, /* Sanitize */ 19921e9f767SBen Widawsky 0x45, /* Persistent Memory Data-at-rest Security */ 20021e9f767SBen Widawsky 0x46, /* Security Passthrough */ 20121e9f767SBen Widawsky }; 20221e9f767SBen Widawsky 20321e9f767SBen Widawsky #define cxl_for_each_cmd(cmd) \ 20421e9f767SBen Widawsky for ((cmd) = &mem_commands[0]; \ 20521e9f767SBen Widawsky ((cmd) - mem_commands) < ARRAY_SIZE(mem_commands); (cmd)++) 20621e9f767SBen Widawsky 20721e9f767SBen Widawsky #define cxl_cmd_count ARRAY_SIZE(mem_commands) 20821e9f767SBen Widawsky 20921e9f767SBen Widawsky static int cxl_mem_wait_for_doorbell(struct cxl_mem *cxlm) 21021e9f767SBen Widawsky { 21121e9f767SBen Widawsky const unsigned long start = jiffies; 21221e9f767SBen Widawsky unsigned long end = start; 21321e9f767SBen Widawsky 21421e9f767SBen Widawsky while (cxl_doorbell_busy(cxlm)) { 21521e9f767SBen Widawsky end = jiffies; 21621e9f767SBen Widawsky 21721e9f767SBen Widawsky if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) { 21821e9f767SBen Widawsky /* Check again in case preempted before timeout test */ 21921e9f767SBen Widawsky if (!cxl_doorbell_busy(cxlm)) 22021e9f767SBen Widawsky break; 22121e9f767SBen Widawsky return -ETIMEDOUT; 22221e9f767SBen Widawsky } 22321e9f767SBen Widawsky cpu_relax(); 22421e9f767SBen Widawsky } 22521e9f767SBen Widawsky 22621e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, "Doorbell wait took %dms", 22721e9f767SBen Widawsky jiffies_to_msecs(end) - jiffies_to_msecs(start)); 22821e9f767SBen Widawsky return 0; 22921e9f767SBen Widawsky } 23021e9f767SBen Widawsky 23121e9f767SBen Widawsky static bool cxl_is_security_command(u16 opcode) 23221e9f767SBen Widawsky { 23321e9f767SBen Widawsky int i; 23421e9f767SBen Widawsky 23521e9f767SBen Widawsky for (i = 0; i < ARRAY_SIZE(security_command_sets); i++) 23621e9f767SBen Widawsky if (security_command_sets[i] == (opcode >> 8)) 23721e9f767SBen Widawsky return true; 23821e9f767SBen Widawsky return false; 23921e9f767SBen Widawsky } 24021e9f767SBen Widawsky 24121e9f767SBen Widawsky static void cxl_mem_mbox_timeout(struct cxl_mem *cxlm, 24221e9f767SBen Widawsky struct mbox_cmd *mbox_cmd) 24321e9f767SBen Widawsky { 24421e9f767SBen Widawsky struct device *dev = &cxlm->pdev->dev; 24521e9f767SBen Widawsky 24621e9f767SBen Widawsky dev_dbg(dev, "Mailbox command (opcode: %#x size: %zub) timed out\n", 24721e9f767SBen Widawsky mbox_cmd->opcode, mbox_cmd->size_in); 24821e9f767SBen Widawsky } 24921e9f767SBen Widawsky 25021e9f767SBen Widawsky /** 25121e9f767SBen Widawsky * __cxl_mem_mbox_send_cmd() - Execute a mailbox command 25221e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 25321e9f767SBen Widawsky * @mbox_cmd: Command to send to the memory device. 25421e9f767SBen Widawsky * 25521e9f767SBen Widawsky * Context: Any context. Expects mbox_mutex to be held. 25621e9f767SBen Widawsky * Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success. 25721e9f767SBen Widawsky * Caller should check the return code in @mbox_cmd to make sure it 25821e9f767SBen Widawsky * succeeded. 25921e9f767SBen Widawsky * 26021e9f767SBen Widawsky * This is a generic form of the CXL mailbox send command thus only using the 26121e9f767SBen Widawsky * registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory 26221e9f767SBen Widawsky * devices, and perhaps other types of CXL devices may have further information 26321e9f767SBen Widawsky * available upon error conditions. Driver facilities wishing to send mailbox 26421e9f767SBen Widawsky * commands should use the wrapper command. 26521e9f767SBen Widawsky * 26621e9f767SBen Widawsky * The CXL spec allows for up to two mailboxes. The intention is for the primary 26721e9f767SBen Widawsky * mailbox to be OS controlled and the secondary mailbox to be used by system 26821e9f767SBen Widawsky * firmware. This allows the OS and firmware to communicate with the device and 26921e9f767SBen Widawsky * not need to coordinate with each other. The driver only uses the primary 27021e9f767SBen Widawsky * mailbox. 27121e9f767SBen Widawsky */ 27221e9f767SBen Widawsky static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, 27321e9f767SBen Widawsky struct mbox_cmd *mbox_cmd) 27421e9f767SBen Widawsky { 27521e9f767SBen Widawsky void __iomem *payload = cxlm->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET; 27621e9f767SBen Widawsky u64 cmd_reg, status_reg; 27721e9f767SBen Widawsky size_t out_len; 27821e9f767SBen Widawsky int rc; 27921e9f767SBen Widawsky 28021e9f767SBen Widawsky lockdep_assert_held(&cxlm->mbox_mutex); 28121e9f767SBen Widawsky 28221e9f767SBen Widawsky /* 28321e9f767SBen Widawsky * Here are the steps from 8.2.8.4 of the CXL 2.0 spec. 28421e9f767SBen Widawsky * 1. Caller reads MB Control Register to verify doorbell is clear 28521e9f767SBen Widawsky * 2. Caller writes Command Register 28621e9f767SBen Widawsky * 3. Caller writes Command Payload Registers if input payload is non-empty 28721e9f767SBen Widawsky * 4. Caller writes MB Control Register to set doorbell 28821e9f767SBen Widawsky * 5. Caller either polls for doorbell to be clear or waits for interrupt if configured 28921e9f767SBen Widawsky * 6. Caller reads MB Status Register to fetch Return code 29021e9f767SBen Widawsky * 7. If command successful, Caller reads Command Register to get Payload Length 29121e9f767SBen Widawsky * 8. If output payload is non-empty, host reads Command Payload Registers 29221e9f767SBen Widawsky * 29321e9f767SBen Widawsky * Hardware is free to do whatever it wants before the doorbell is rung, 29421e9f767SBen Widawsky * and isn't allowed to change anything after it clears the doorbell. As 29521e9f767SBen Widawsky * such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can 29621e9f767SBen Widawsky * also happen in any order (though some orders might not make sense). 29721e9f767SBen Widawsky */ 29821e9f767SBen Widawsky 29921e9f767SBen Widawsky /* #1 */ 30021e9f767SBen Widawsky if (cxl_doorbell_busy(cxlm)) { 30121e9f767SBen Widawsky dev_err_ratelimited(&cxlm->pdev->dev, 30221e9f767SBen Widawsky "Mailbox re-busy after acquiring\n"); 30321e9f767SBen Widawsky return -EBUSY; 30421e9f767SBen Widawsky } 30521e9f767SBen Widawsky 30621e9f767SBen Widawsky cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK, 30721e9f767SBen Widawsky mbox_cmd->opcode); 30821e9f767SBen Widawsky if (mbox_cmd->size_in) { 30921e9f767SBen Widawsky if (WARN_ON(!mbox_cmd->payload_in)) 31021e9f767SBen Widawsky return -EINVAL; 31121e9f767SBen Widawsky 31221e9f767SBen Widawsky cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, 31321e9f767SBen Widawsky mbox_cmd->size_in); 31421e9f767SBen Widawsky memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in); 31521e9f767SBen Widawsky } 31621e9f767SBen Widawsky 31721e9f767SBen Widawsky /* #2, #3 */ 31821e9f767SBen Widawsky writeq(cmd_reg, cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 31921e9f767SBen Widawsky 32021e9f767SBen Widawsky /* #4 */ 32121e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, "Sending command\n"); 32221e9f767SBen Widawsky writel(CXLDEV_MBOX_CTRL_DOORBELL, 32321e9f767SBen Widawsky cxlm->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET); 32421e9f767SBen Widawsky 32521e9f767SBen Widawsky /* #5 */ 32621e9f767SBen Widawsky rc = cxl_mem_wait_for_doorbell(cxlm); 32721e9f767SBen Widawsky if (rc == -ETIMEDOUT) { 32821e9f767SBen Widawsky cxl_mem_mbox_timeout(cxlm, mbox_cmd); 32921e9f767SBen Widawsky return rc; 33021e9f767SBen Widawsky } 33121e9f767SBen Widawsky 33221e9f767SBen Widawsky /* #6 */ 33321e9f767SBen Widawsky status_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET); 33421e9f767SBen Widawsky mbox_cmd->return_code = 33521e9f767SBen Widawsky FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg); 33621e9f767SBen Widawsky 33721e9f767SBen Widawsky if (mbox_cmd->return_code != 0) { 33821e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, "Mailbox operation had an error\n"); 33921e9f767SBen Widawsky return 0; 34021e9f767SBen Widawsky } 34121e9f767SBen Widawsky 34221e9f767SBen Widawsky /* #7 */ 34321e9f767SBen Widawsky cmd_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 34421e9f767SBen Widawsky out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg); 34521e9f767SBen Widawsky 34621e9f767SBen Widawsky /* #8 */ 34721e9f767SBen Widawsky if (out_len && mbox_cmd->payload_out) { 34821e9f767SBen Widawsky /* 34921e9f767SBen Widawsky * Sanitize the copy. If hardware misbehaves, out_len per the 35021e9f767SBen Widawsky * spec can actually be greater than the max allowed size (21 35121e9f767SBen Widawsky * bits available but spec defined 1M max). The caller also may 35221e9f767SBen Widawsky * have requested less data than the hardware supplied even 35321e9f767SBen Widawsky * within spec. 35421e9f767SBen Widawsky */ 35521e9f767SBen Widawsky size_t n = min3(mbox_cmd->size_out, cxlm->payload_size, out_len); 35621e9f767SBen Widawsky 35721e9f767SBen Widawsky memcpy_fromio(mbox_cmd->payload_out, payload, n); 35821e9f767SBen Widawsky mbox_cmd->size_out = n; 35921e9f767SBen Widawsky } else { 36021e9f767SBen Widawsky mbox_cmd->size_out = 0; 36121e9f767SBen Widawsky } 36221e9f767SBen Widawsky 36321e9f767SBen Widawsky return 0; 36421e9f767SBen Widawsky } 36521e9f767SBen Widawsky 36621e9f767SBen Widawsky /** 36721e9f767SBen Widawsky * cxl_mem_mbox_get() - Acquire exclusive access to the mailbox. 36821e9f767SBen Widawsky * @cxlm: The memory device to gain access to. 36921e9f767SBen Widawsky * 37021e9f767SBen Widawsky * Context: Any context. Takes the mbox_mutex. 37121e9f767SBen Widawsky * Return: 0 if exclusive access was acquired. 37221e9f767SBen Widawsky */ 37321e9f767SBen Widawsky static int cxl_mem_mbox_get(struct cxl_mem *cxlm) 37421e9f767SBen Widawsky { 37521e9f767SBen Widawsky struct device *dev = &cxlm->pdev->dev; 37621e9f767SBen Widawsky u64 md_status; 37721e9f767SBen Widawsky int rc; 37821e9f767SBen Widawsky 37921e9f767SBen Widawsky mutex_lock_io(&cxlm->mbox_mutex); 38021e9f767SBen Widawsky 38121e9f767SBen Widawsky /* 38221e9f767SBen Widawsky * XXX: There is some amount of ambiguity in the 2.0 version of the spec 38321e9f767SBen Widawsky * around the mailbox interface ready (8.2.8.5.1.1). The purpose of the 38421e9f767SBen Widawsky * bit is to allow firmware running on the device to notify the driver 38521e9f767SBen Widawsky * that it's ready to receive commands. It is unclear if the bit needs 38621e9f767SBen Widawsky * to be read for each transaction mailbox, ie. the firmware can switch 38721e9f767SBen Widawsky * it on and off as needed. Second, there is no defined timeout for 38821e9f767SBen Widawsky * mailbox ready, like there is for the doorbell interface. 38921e9f767SBen Widawsky * 39021e9f767SBen Widawsky * Assumptions: 39121e9f767SBen Widawsky * 1. The firmware might toggle the Mailbox Interface Ready bit, check 39221e9f767SBen Widawsky * it for every command. 39321e9f767SBen Widawsky * 39421e9f767SBen Widawsky * 2. If the doorbell is clear, the firmware should have first set the 39521e9f767SBen Widawsky * Mailbox Interface Ready bit. Therefore, waiting for the doorbell 39621e9f767SBen Widawsky * to be ready is sufficient. 39721e9f767SBen Widawsky */ 39821e9f767SBen Widawsky rc = cxl_mem_wait_for_doorbell(cxlm); 39921e9f767SBen Widawsky if (rc) { 40021e9f767SBen Widawsky dev_warn(dev, "Mailbox interface not ready\n"); 40121e9f767SBen Widawsky goto out; 40221e9f767SBen Widawsky } 40321e9f767SBen Widawsky 40421e9f767SBen Widawsky md_status = readq(cxlm->regs.memdev + CXLMDEV_STATUS_OFFSET); 40521e9f767SBen Widawsky if (!(md_status & CXLMDEV_MBOX_IF_READY && CXLMDEV_READY(md_status))) { 40621e9f767SBen Widawsky dev_err(dev, "mbox: reported doorbell ready, but not mbox ready\n"); 40721e9f767SBen Widawsky rc = -EBUSY; 40821e9f767SBen Widawsky goto out; 40921e9f767SBen Widawsky } 41021e9f767SBen Widawsky 41121e9f767SBen Widawsky /* 41221e9f767SBen Widawsky * Hardware shouldn't allow a ready status but also have failure bits 41321e9f767SBen Widawsky * set. Spit out an error, this should be a bug report 41421e9f767SBen Widawsky */ 41521e9f767SBen Widawsky rc = -EFAULT; 41621e9f767SBen Widawsky if (md_status & CXLMDEV_DEV_FATAL) { 41721e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but fatal\n"); 41821e9f767SBen Widawsky goto out; 41921e9f767SBen Widawsky } 42021e9f767SBen Widawsky if (md_status & CXLMDEV_FW_HALT) { 42121e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but halted\n"); 42221e9f767SBen Widawsky goto out; 42321e9f767SBen Widawsky } 42421e9f767SBen Widawsky if (CXLMDEV_RESET_NEEDED(md_status)) { 42521e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but reset needed\n"); 42621e9f767SBen Widawsky goto out; 42721e9f767SBen Widawsky } 42821e9f767SBen Widawsky 42921e9f767SBen Widawsky /* with lock held */ 43021e9f767SBen Widawsky return 0; 43121e9f767SBen Widawsky 43221e9f767SBen Widawsky out: 43321e9f767SBen Widawsky mutex_unlock(&cxlm->mbox_mutex); 43421e9f767SBen Widawsky return rc; 43521e9f767SBen Widawsky } 43621e9f767SBen Widawsky 43721e9f767SBen Widawsky /** 43821e9f767SBen Widawsky * cxl_mem_mbox_put() - Release exclusive access to the mailbox. 43921e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 44021e9f767SBen Widawsky * 44121e9f767SBen Widawsky * Context: Any context. Expects mbox_mutex to be held. 44221e9f767SBen Widawsky */ 44321e9f767SBen Widawsky static void cxl_mem_mbox_put(struct cxl_mem *cxlm) 44421e9f767SBen Widawsky { 44521e9f767SBen Widawsky mutex_unlock(&cxlm->mbox_mutex); 44621e9f767SBen Widawsky } 44721e9f767SBen Widawsky 44821e9f767SBen Widawsky /** 44921e9f767SBen Widawsky * handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace. 45021e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 45121e9f767SBen Widawsky * @cmd: The validated command. 45221e9f767SBen Widawsky * @in_payload: Pointer to userspace's input payload. 45321e9f767SBen Widawsky * @out_payload: Pointer to userspace's output payload. 45421e9f767SBen Widawsky * @size_out: (Input) Max payload size to copy out. 45521e9f767SBen Widawsky * (Output) Payload size hardware generated. 45621e9f767SBen Widawsky * @retval: Hardware generated return code from the operation. 45721e9f767SBen Widawsky * 45821e9f767SBen Widawsky * Return: 45921e9f767SBen Widawsky * * %0 - Mailbox transaction succeeded. This implies the mailbox 46021e9f767SBen Widawsky * protocol completed successfully not that the operation itself 46121e9f767SBen Widawsky * was successful. 46221e9f767SBen Widawsky * * %-ENOMEM - Couldn't allocate a bounce buffer. 46321e9f767SBen Widawsky * * %-EFAULT - Something happened with copy_to/from_user. 46421e9f767SBen Widawsky * * %-EINTR - Mailbox acquisition interrupted. 46521e9f767SBen Widawsky * * %-EXXX - Transaction level failures. 46621e9f767SBen Widawsky * 46721e9f767SBen Widawsky * Creates the appropriate mailbox command and dispatches it on behalf of a 46821e9f767SBen Widawsky * userspace request. The input and output payloads are copied between 46921e9f767SBen Widawsky * userspace. 47021e9f767SBen Widawsky * 47121e9f767SBen Widawsky * See cxl_send_cmd(). 47221e9f767SBen Widawsky */ 47321e9f767SBen Widawsky static int handle_mailbox_cmd_from_user(struct cxl_mem *cxlm, 47421e9f767SBen Widawsky const struct cxl_mem_command *cmd, 47521e9f767SBen Widawsky u64 in_payload, u64 out_payload, 47621e9f767SBen Widawsky s32 *size_out, u32 *retval) 47721e9f767SBen Widawsky { 47821e9f767SBen Widawsky struct device *dev = &cxlm->pdev->dev; 47921e9f767SBen Widawsky struct mbox_cmd mbox_cmd = { 48021e9f767SBen Widawsky .opcode = cmd->opcode, 48121e9f767SBen Widawsky .size_in = cmd->info.size_in, 48221e9f767SBen Widawsky .size_out = cmd->info.size_out, 48321e9f767SBen Widawsky }; 48421e9f767SBen Widawsky int rc; 48521e9f767SBen Widawsky 48621e9f767SBen Widawsky if (cmd->info.size_out) { 48721e9f767SBen Widawsky mbox_cmd.payload_out = kvzalloc(cmd->info.size_out, GFP_KERNEL); 48821e9f767SBen Widawsky if (!mbox_cmd.payload_out) 48921e9f767SBen Widawsky return -ENOMEM; 49021e9f767SBen Widawsky } 49121e9f767SBen Widawsky 49221e9f767SBen Widawsky if (cmd->info.size_in) { 49321e9f767SBen Widawsky mbox_cmd.payload_in = vmemdup_user(u64_to_user_ptr(in_payload), 49421e9f767SBen Widawsky cmd->info.size_in); 49521e9f767SBen Widawsky if (IS_ERR(mbox_cmd.payload_in)) { 49621e9f767SBen Widawsky kvfree(mbox_cmd.payload_out); 49721e9f767SBen Widawsky return PTR_ERR(mbox_cmd.payload_in); 49821e9f767SBen Widawsky } 49921e9f767SBen Widawsky } 50021e9f767SBen Widawsky 50121e9f767SBen Widawsky rc = cxl_mem_mbox_get(cxlm); 50221e9f767SBen Widawsky if (rc) 50321e9f767SBen Widawsky goto out; 50421e9f767SBen Widawsky 50521e9f767SBen Widawsky dev_dbg(dev, 50621e9f767SBen Widawsky "Submitting %s command for user\n" 50721e9f767SBen Widawsky "\topcode: %x\n" 50821e9f767SBen Widawsky "\tsize: %ub\n", 50921e9f767SBen Widawsky cxl_command_names[cmd->info.id].name, mbox_cmd.opcode, 51021e9f767SBen Widawsky cmd->info.size_in); 51121e9f767SBen Widawsky 51221e9f767SBen Widawsky dev_WARN_ONCE(dev, cmd->info.id == CXL_MEM_COMMAND_ID_RAW, 51321e9f767SBen Widawsky "raw command path used\n"); 51421e9f767SBen Widawsky 51521e9f767SBen Widawsky rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd); 51621e9f767SBen Widawsky cxl_mem_mbox_put(cxlm); 51721e9f767SBen Widawsky if (rc) 51821e9f767SBen Widawsky goto out; 51921e9f767SBen Widawsky 52021e9f767SBen Widawsky /* 52121e9f767SBen Widawsky * @size_out contains the max size that's allowed to be written back out 52221e9f767SBen Widawsky * to userspace. While the payload may have written more output than 52321e9f767SBen Widawsky * this it will have to be ignored. 52421e9f767SBen Widawsky */ 52521e9f767SBen Widawsky if (mbox_cmd.size_out) { 52621e9f767SBen Widawsky dev_WARN_ONCE(dev, mbox_cmd.size_out > *size_out, 52721e9f767SBen Widawsky "Invalid return size\n"); 52821e9f767SBen Widawsky if (copy_to_user(u64_to_user_ptr(out_payload), 52921e9f767SBen Widawsky mbox_cmd.payload_out, mbox_cmd.size_out)) { 53021e9f767SBen Widawsky rc = -EFAULT; 53121e9f767SBen Widawsky goto out; 53221e9f767SBen Widawsky } 53321e9f767SBen Widawsky } 53421e9f767SBen Widawsky 53521e9f767SBen Widawsky *size_out = mbox_cmd.size_out; 53621e9f767SBen Widawsky *retval = mbox_cmd.return_code; 53721e9f767SBen Widawsky 53821e9f767SBen Widawsky out: 53921e9f767SBen Widawsky kvfree(mbox_cmd.payload_in); 54021e9f767SBen Widawsky kvfree(mbox_cmd.payload_out); 54121e9f767SBen Widawsky return rc; 54221e9f767SBen Widawsky } 54321e9f767SBen Widawsky 54421e9f767SBen Widawsky static bool cxl_mem_raw_command_allowed(u16 opcode) 54521e9f767SBen Widawsky { 54621e9f767SBen Widawsky int i; 54721e9f767SBen Widawsky 54821e9f767SBen Widawsky if (!IS_ENABLED(CONFIG_CXL_MEM_RAW_COMMANDS)) 54921e9f767SBen Widawsky return false; 55021e9f767SBen Widawsky 55121e9f767SBen Widawsky if (security_locked_down(LOCKDOWN_NONE)) 55221e9f767SBen Widawsky return false; 55321e9f767SBen Widawsky 55421e9f767SBen Widawsky if (cxl_raw_allow_all) 55521e9f767SBen Widawsky return true; 55621e9f767SBen Widawsky 55721e9f767SBen Widawsky if (cxl_is_security_command(opcode)) 55821e9f767SBen Widawsky return false; 55921e9f767SBen Widawsky 56021e9f767SBen Widawsky for (i = 0; i < ARRAY_SIZE(cxl_disabled_raw_commands); i++) 56121e9f767SBen Widawsky if (cxl_disabled_raw_commands[i] == opcode) 56221e9f767SBen Widawsky return false; 56321e9f767SBen Widawsky 56421e9f767SBen Widawsky return true; 56521e9f767SBen Widawsky } 56621e9f767SBen Widawsky 56721e9f767SBen Widawsky /** 56821e9f767SBen Widawsky * cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND. 56921e9f767SBen Widawsky * @cxlm: &struct cxl_mem device whose mailbox will be used. 57021e9f767SBen Widawsky * @send_cmd: &struct cxl_send_command copied in from userspace. 57121e9f767SBen Widawsky * @out_cmd: Sanitized and populated &struct cxl_mem_command. 57221e9f767SBen Widawsky * 57321e9f767SBen Widawsky * Return: 57421e9f767SBen Widawsky * * %0 - @out_cmd is ready to send. 57521e9f767SBen Widawsky * * %-ENOTTY - Invalid command specified. 57621e9f767SBen Widawsky * * %-EINVAL - Reserved fields or invalid values were used. 57721e9f767SBen Widawsky * * %-ENOMEM - Input or output buffer wasn't sized properly. 57821e9f767SBen Widawsky * * %-EPERM - Attempted to use a protected command. 57921e9f767SBen Widawsky * 58021e9f767SBen Widawsky * The result of this command is a fully validated command in @out_cmd that is 58121e9f767SBen Widawsky * safe to send to the hardware. 58221e9f767SBen Widawsky * 58321e9f767SBen Widawsky * See handle_mailbox_cmd_from_user() 58421e9f767SBen Widawsky */ 58521e9f767SBen Widawsky static int cxl_validate_cmd_from_user(struct cxl_mem *cxlm, 58621e9f767SBen Widawsky const struct cxl_send_command *send_cmd, 58721e9f767SBen Widawsky struct cxl_mem_command *out_cmd) 58821e9f767SBen Widawsky { 58921e9f767SBen Widawsky const struct cxl_command_info *info; 59021e9f767SBen Widawsky struct cxl_mem_command *c; 59121e9f767SBen Widawsky 59221e9f767SBen Widawsky if (send_cmd->id == 0 || send_cmd->id >= CXL_MEM_COMMAND_ID_MAX) 59321e9f767SBen Widawsky return -ENOTTY; 59421e9f767SBen Widawsky 59521e9f767SBen Widawsky /* 59621e9f767SBen Widawsky * The user can never specify an input payload larger than what hardware 59721e9f767SBen Widawsky * supports, but output can be arbitrarily large (simply write out as 59821e9f767SBen Widawsky * much data as the hardware provides). 59921e9f767SBen Widawsky */ 60021e9f767SBen Widawsky if (send_cmd->in.size > cxlm->payload_size) 60121e9f767SBen Widawsky return -EINVAL; 60221e9f767SBen Widawsky 60321e9f767SBen Widawsky /* 60421e9f767SBen Widawsky * Checks are bypassed for raw commands but a WARN/taint will occur 60521e9f767SBen Widawsky * later in the callchain 60621e9f767SBen Widawsky */ 60721e9f767SBen Widawsky if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW) { 60821e9f767SBen Widawsky const struct cxl_mem_command temp = { 60921e9f767SBen Widawsky .info = { 61021e9f767SBen Widawsky .id = CXL_MEM_COMMAND_ID_RAW, 61121e9f767SBen Widawsky .flags = 0, 61221e9f767SBen Widawsky .size_in = send_cmd->in.size, 61321e9f767SBen Widawsky .size_out = send_cmd->out.size, 61421e9f767SBen Widawsky }, 61521e9f767SBen Widawsky .opcode = send_cmd->raw.opcode 61621e9f767SBen Widawsky }; 61721e9f767SBen Widawsky 61821e9f767SBen Widawsky if (send_cmd->raw.rsvd) 61921e9f767SBen Widawsky return -EINVAL; 62021e9f767SBen Widawsky 62121e9f767SBen Widawsky /* 62221e9f767SBen Widawsky * Unlike supported commands, the output size of RAW commands 62321e9f767SBen Widawsky * gets passed along without further checking, so it must be 62421e9f767SBen Widawsky * validated here. 62521e9f767SBen Widawsky */ 62621e9f767SBen Widawsky if (send_cmd->out.size > cxlm->payload_size) 62721e9f767SBen Widawsky return -EINVAL; 62821e9f767SBen Widawsky 62921e9f767SBen Widawsky if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode)) 63021e9f767SBen Widawsky return -EPERM; 63121e9f767SBen Widawsky 63221e9f767SBen Widawsky memcpy(out_cmd, &temp, sizeof(temp)); 63321e9f767SBen Widawsky 63421e9f767SBen Widawsky return 0; 63521e9f767SBen Widawsky } 63621e9f767SBen Widawsky 63721e9f767SBen Widawsky if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK) 63821e9f767SBen Widawsky return -EINVAL; 63921e9f767SBen Widawsky 64021e9f767SBen Widawsky if (send_cmd->rsvd) 64121e9f767SBen Widawsky return -EINVAL; 64221e9f767SBen Widawsky 64321e9f767SBen Widawsky if (send_cmd->in.rsvd || send_cmd->out.rsvd) 64421e9f767SBen Widawsky return -EINVAL; 64521e9f767SBen Widawsky 64621e9f767SBen Widawsky /* Convert user's command into the internal representation */ 64721e9f767SBen Widawsky c = &mem_commands[send_cmd->id]; 64821e9f767SBen Widawsky info = &c->info; 64921e9f767SBen Widawsky 65021e9f767SBen Widawsky /* Check that the command is enabled for hardware */ 65121e9f767SBen Widawsky if (!test_bit(info->id, cxlm->enabled_cmds)) 65221e9f767SBen Widawsky return -ENOTTY; 65321e9f767SBen Widawsky 65421e9f767SBen Widawsky /* Check the input buffer is the expected size */ 65521e9f767SBen Widawsky if (info->size_in >= 0 && info->size_in != send_cmd->in.size) 65621e9f767SBen Widawsky return -ENOMEM; 65721e9f767SBen Widawsky 65821e9f767SBen Widawsky /* Check the output buffer is at least large enough */ 65921e9f767SBen Widawsky if (info->size_out >= 0 && send_cmd->out.size < info->size_out) 66021e9f767SBen Widawsky return -ENOMEM; 66121e9f767SBen Widawsky 66221e9f767SBen Widawsky memcpy(out_cmd, c, sizeof(*c)); 66321e9f767SBen Widawsky out_cmd->info.size_in = send_cmd->in.size; 66421e9f767SBen Widawsky /* 66521e9f767SBen Widawsky * XXX: out_cmd->info.size_out will be controlled by the driver, and the 66621e9f767SBen Widawsky * specified number of bytes @send_cmd->out.size will be copied back out 66721e9f767SBen Widawsky * to userspace. 66821e9f767SBen Widawsky */ 66921e9f767SBen Widawsky 67021e9f767SBen Widawsky return 0; 67121e9f767SBen Widawsky } 67221e9f767SBen Widawsky 67321e9f767SBen Widawsky static int cxl_query_cmd(struct cxl_memdev *cxlmd, 67421e9f767SBen Widawsky struct cxl_mem_query_commands __user *q) 67521e9f767SBen Widawsky { 67621e9f767SBen Widawsky struct device *dev = &cxlmd->dev; 67721e9f767SBen Widawsky struct cxl_mem_command *cmd; 67821e9f767SBen Widawsky u32 n_commands; 67921e9f767SBen Widawsky int j = 0; 68021e9f767SBen Widawsky 68121e9f767SBen Widawsky dev_dbg(dev, "Query IOCTL\n"); 68221e9f767SBen Widawsky 68321e9f767SBen Widawsky if (get_user(n_commands, &q->n_commands)) 68421e9f767SBen Widawsky return -EFAULT; 68521e9f767SBen Widawsky 68621e9f767SBen Widawsky /* returns the total number if 0 elements are requested. */ 68721e9f767SBen Widawsky if (n_commands == 0) 68821e9f767SBen Widawsky return put_user(cxl_cmd_count, &q->n_commands); 68921e9f767SBen Widawsky 69021e9f767SBen Widawsky /* 69121e9f767SBen Widawsky * otherwise, return max(n_commands, total commands) cxl_command_info 69221e9f767SBen Widawsky * structures. 69321e9f767SBen Widawsky */ 69421e9f767SBen Widawsky cxl_for_each_cmd(cmd) { 69521e9f767SBen Widawsky const struct cxl_command_info *info = &cmd->info; 69621e9f767SBen Widawsky 69721e9f767SBen Widawsky if (copy_to_user(&q->commands[j++], info, sizeof(*info))) 69821e9f767SBen Widawsky return -EFAULT; 69921e9f767SBen Widawsky 70021e9f767SBen Widawsky if (j == n_commands) 70121e9f767SBen Widawsky break; 70221e9f767SBen Widawsky } 70321e9f767SBen Widawsky 70421e9f767SBen Widawsky return 0; 70521e9f767SBen Widawsky } 70621e9f767SBen Widawsky 70721e9f767SBen Widawsky static int cxl_send_cmd(struct cxl_memdev *cxlmd, 70821e9f767SBen Widawsky struct cxl_send_command __user *s) 70921e9f767SBen Widawsky { 71021e9f767SBen Widawsky struct cxl_mem *cxlm = cxlmd->cxlm; 71121e9f767SBen Widawsky struct device *dev = &cxlmd->dev; 71221e9f767SBen Widawsky struct cxl_send_command send; 71321e9f767SBen Widawsky struct cxl_mem_command c; 71421e9f767SBen Widawsky int rc; 71521e9f767SBen Widawsky 71621e9f767SBen Widawsky dev_dbg(dev, "Send IOCTL\n"); 71721e9f767SBen Widawsky 71821e9f767SBen Widawsky if (copy_from_user(&send, s, sizeof(send))) 71921e9f767SBen Widawsky return -EFAULT; 72021e9f767SBen Widawsky 72121e9f767SBen Widawsky rc = cxl_validate_cmd_from_user(cxlmd->cxlm, &send, &c); 72221e9f767SBen Widawsky if (rc) 72321e9f767SBen Widawsky return rc; 72421e9f767SBen Widawsky 72521e9f767SBen Widawsky /* Prepare to handle a full payload for variable sized output */ 72621e9f767SBen Widawsky if (c.info.size_out < 0) 72721e9f767SBen Widawsky c.info.size_out = cxlm->payload_size; 72821e9f767SBen Widawsky 72921e9f767SBen Widawsky rc = handle_mailbox_cmd_from_user(cxlm, &c, send.in.payload, 73021e9f767SBen Widawsky send.out.payload, &send.out.size, 73121e9f767SBen Widawsky &send.retval); 73221e9f767SBen Widawsky if (rc) 73321e9f767SBen Widawsky return rc; 73421e9f767SBen Widawsky 73521e9f767SBen Widawsky if (copy_to_user(s, &send, sizeof(send))) 73621e9f767SBen Widawsky return -EFAULT; 73721e9f767SBen Widawsky 73821e9f767SBen Widawsky return 0; 73921e9f767SBen Widawsky } 74021e9f767SBen Widawsky 74121e9f767SBen Widawsky static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, 74221e9f767SBen Widawsky unsigned long arg) 74321e9f767SBen Widawsky { 74421e9f767SBen Widawsky switch (cmd) { 74521e9f767SBen Widawsky case CXL_MEM_QUERY_COMMANDS: 74621e9f767SBen Widawsky return cxl_query_cmd(cxlmd, (void __user *)arg); 74721e9f767SBen Widawsky case CXL_MEM_SEND_COMMAND: 74821e9f767SBen Widawsky return cxl_send_cmd(cxlmd, (void __user *)arg); 74921e9f767SBen Widawsky default: 75021e9f767SBen Widawsky return -ENOTTY; 75121e9f767SBen Widawsky } 75221e9f767SBen Widawsky } 75321e9f767SBen Widawsky 75421e9f767SBen Widawsky static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, 75521e9f767SBen Widawsky unsigned long arg) 75621e9f767SBen Widawsky { 75721e9f767SBen Widawsky struct cxl_memdev *cxlmd = file->private_data; 75821e9f767SBen Widawsky int rc = -ENXIO; 75921e9f767SBen Widawsky 76021e9f767SBen Widawsky down_read(&cxl_memdev_rwsem); 76121e9f767SBen Widawsky if (cxlmd->cxlm) 76221e9f767SBen Widawsky rc = __cxl_memdev_ioctl(cxlmd, cmd, arg); 76321e9f767SBen Widawsky up_read(&cxl_memdev_rwsem); 76421e9f767SBen Widawsky 76521e9f767SBen Widawsky return rc; 76621e9f767SBen Widawsky } 76721e9f767SBen Widawsky 76821e9f767SBen Widawsky static int cxl_memdev_open(struct inode *inode, struct file *file) 76921e9f767SBen Widawsky { 77021e9f767SBen Widawsky struct cxl_memdev *cxlmd = 77121e9f767SBen Widawsky container_of(inode->i_cdev, typeof(*cxlmd), cdev); 77221e9f767SBen Widawsky 77321e9f767SBen Widawsky get_device(&cxlmd->dev); 77421e9f767SBen Widawsky file->private_data = cxlmd; 77521e9f767SBen Widawsky 77621e9f767SBen Widawsky return 0; 77721e9f767SBen Widawsky } 77821e9f767SBen Widawsky 77921e9f767SBen Widawsky static int cxl_memdev_release_file(struct inode *inode, struct file *file) 78021e9f767SBen Widawsky { 78121e9f767SBen Widawsky struct cxl_memdev *cxlmd = 78221e9f767SBen Widawsky container_of(inode->i_cdev, typeof(*cxlmd), cdev); 78321e9f767SBen Widawsky 78421e9f767SBen Widawsky put_device(&cxlmd->dev); 78521e9f767SBen Widawsky 78621e9f767SBen Widawsky return 0; 78721e9f767SBen Widawsky } 78821e9f767SBen Widawsky 78921e9f767SBen Widawsky static const struct file_operations cxl_memdev_fops = { 79021e9f767SBen Widawsky .owner = THIS_MODULE, 79121e9f767SBen Widawsky .unlocked_ioctl = cxl_memdev_ioctl, 79221e9f767SBen Widawsky .open = cxl_memdev_open, 79321e9f767SBen Widawsky .release = cxl_memdev_release_file, 79421e9f767SBen Widawsky .compat_ioctl = compat_ptr_ioctl, 79521e9f767SBen Widawsky .llseek = noop_llseek, 79621e9f767SBen Widawsky }; 79721e9f767SBen Widawsky 79821e9f767SBen Widawsky static inline struct cxl_mem_command *cxl_mem_find_command(u16 opcode) 79921e9f767SBen Widawsky { 80021e9f767SBen Widawsky struct cxl_mem_command *c; 80121e9f767SBen Widawsky 80221e9f767SBen Widawsky cxl_for_each_cmd(c) 80321e9f767SBen Widawsky if (c->opcode == opcode) 80421e9f767SBen Widawsky return c; 80521e9f767SBen Widawsky 80621e9f767SBen Widawsky return NULL; 80721e9f767SBen Widawsky } 80821e9f767SBen Widawsky 80921e9f767SBen Widawsky /** 81021e9f767SBen Widawsky * cxl_mem_mbox_send_cmd() - Send a mailbox command to a memory device. 81121e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 81221e9f767SBen Widawsky * @opcode: Opcode for the mailbox command. 81321e9f767SBen Widawsky * @in: The input payload for the mailbox command. 81421e9f767SBen Widawsky * @in_size: The length of the input payload 81521e9f767SBen Widawsky * @out: Caller allocated buffer for the output. 81621e9f767SBen Widawsky * @out_size: Expected size of output. 81721e9f767SBen Widawsky * 81821e9f767SBen Widawsky * Context: Any context. Will acquire and release mbox_mutex. 81921e9f767SBen Widawsky * Return: 82021e9f767SBen Widawsky * * %>=0 - Number of bytes returned in @out. 82121e9f767SBen Widawsky * * %-E2BIG - Payload is too large for hardware. 82221e9f767SBen Widawsky * * %-EBUSY - Couldn't acquire exclusive mailbox access. 82321e9f767SBen Widawsky * * %-EFAULT - Hardware error occurred. 82421e9f767SBen Widawsky * * %-ENXIO - Command completed, but device reported an error. 82521e9f767SBen Widawsky * * %-EIO - Unexpected output size. 82621e9f767SBen Widawsky * 82721e9f767SBen Widawsky * Mailbox commands may execute successfully yet the device itself reported an 82821e9f767SBen Widawsky * error. While this distinction can be useful for commands from userspace, the 82921e9f767SBen Widawsky * kernel will only be able to use results when both are successful. 83021e9f767SBen Widawsky * 83121e9f767SBen Widawsky * See __cxl_mem_mbox_send_cmd() 83221e9f767SBen Widawsky */ 83321e9f767SBen Widawsky static int cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, u16 opcode, 83421e9f767SBen Widawsky void *in, size_t in_size, 83521e9f767SBen Widawsky void *out, size_t out_size) 83621e9f767SBen Widawsky { 83721e9f767SBen Widawsky const struct cxl_mem_command *cmd = cxl_mem_find_command(opcode); 83821e9f767SBen Widawsky struct mbox_cmd mbox_cmd = { 83921e9f767SBen Widawsky .opcode = opcode, 84021e9f767SBen Widawsky .payload_in = in, 84121e9f767SBen Widawsky .size_in = in_size, 84221e9f767SBen Widawsky .size_out = out_size, 84321e9f767SBen Widawsky .payload_out = out, 84421e9f767SBen Widawsky }; 84521e9f767SBen Widawsky int rc; 84621e9f767SBen Widawsky 84721e9f767SBen Widawsky if (out_size > cxlm->payload_size) 84821e9f767SBen Widawsky return -E2BIG; 84921e9f767SBen Widawsky 85021e9f767SBen Widawsky rc = cxl_mem_mbox_get(cxlm); 85121e9f767SBen Widawsky if (rc) 85221e9f767SBen Widawsky return rc; 85321e9f767SBen Widawsky 85421e9f767SBen Widawsky rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd); 85521e9f767SBen Widawsky cxl_mem_mbox_put(cxlm); 85621e9f767SBen Widawsky if (rc) 85721e9f767SBen Widawsky return rc; 85821e9f767SBen Widawsky 85921e9f767SBen Widawsky /* TODO: Map return code to proper kernel style errno */ 86021e9f767SBen Widawsky if (mbox_cmd.return_code != CXL_MBOX_SUCCESS) 86121e9f767SBen Widawsky return -ENXIO; 86221e9f767SBen Widawsky 86321e9f767SBen Widawsky /* 86421e9f767SBen Widawsky * Variable sized commands can't be validated and so it's up to the 86521e9f767SBen Widawsky * caller to do that if they wish. 86621e9f767SBen Widawsky */ 86721e9f767SBen Widawsky if (cmd->info.size_out >= 0 && mbox_cmd.size_out != out_size) 86821e9f767SBen Widawsky return -EIO; 86921e9f767SBen Widawsky 87021e9f767SBen Widawsky return 0; 87121e9f767SBen Widawsky } 87221e9f767SBen Widawsky 87321e9f767SBen Widawsky /** 87421e9f767SBen Widawsky * cxl_mem_setup_regs() - Setup necessary MMIO. 87521e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 87621e9f767SBen Widawsky * 87721e9f767SBen Widawsky * Return: 0 if all necessary registers mapped. 87821e9f767SBen Widawsky * 87921e9f767SBen Widawsky * A memory device is required by spec to implement a certain set of MMIO 88021e9f767SBen Widawsky * regions. The purpose of this function is to enumerate and map those 88121e9f767SBen Widawsky * registers. 88221e9f767SBen Widawsky */ 88321e9f767SBen Widawsky static int cxl_mem_setup_regs(struct cxl_mem *cxlm) 88421e9f767SBen Widawsky { 88521e9f767SBen Widawsky struct device *dev = &cxlm->pdev->dev; 88621e9f767SBen Widawsky struct cxl_regs *regs = &cxlm->regs; 88721e9f767SBen Widawsky 88821e9f767SBen Widawsky cxl_setup_device_regs(dev, cxlm->base, ®s->device_regs); 88921e9f767SBen Widawsky 89021e9f767SBen Widawsky if (!regs->status || !regs->mbox || !regs->memdev) { 89121e9f767SBen Widawsky dev_err(dev, "registers not found: %s%s%s\n", 89221e9f767SBen Widawsky !regs->status ? "status " : "", 89321e9f767SBen Widawsky !regs->mbox ? "mbox " : "", 89421e9f767SBen Widawsky !regs->memdev ? "memdev" : ""); 89521e9f767SBen Widawsky return -ENXIO; 89621e9f767SBen Widawsky } 89721e9f767SBen Widawsky 89821e9f767SBen Widawsky return 0; 89921e9f767SBen Widawsky } 90021e9f767SBen Widawsky 90121e9f767SBen Widawsky static int cxl_mem_setup_mailbox(struct cxl_mem *cxlm) 90221e9f767SBen Widawsky { 90321e9f767SBen Widawsky const int cap = readl(cxlm->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET); 90421e9f767SBen Widawsky 90521e9f767SBen Widawsky cxlm->payload_size = 90621e9f767SBen Widawsky 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap); 90721e9f767SBen Widawsky 90821e9f767SBen Widawsky /* 90921e9f767SBen Widawsky * CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register 91021e9f767SBen Widawsky * 91121e9f767SBen Widawsky * If the size is too small, mandatory commands will not work and so 91221e9f767SBen Widawsky * there's no point in going forward. If the size is too large, there's 91321e9f767SBen Widawsky * no harm is soft limiting it. 91421e9f767SBen Widawsky */ 91521e9f767SBen Widawsky cxlm->payload_size = min_t(size_t, cxlm->payload_size, SZ_1M); 91621e9f767SBen Widawsky if (cxlm->payload_size < 256) { 91721e9f767SBen Widawsky dev_err(&cxlm->pdev->dev, "Mailbox is too small (%zub)", 91821e9f767SBen Widawsky cxlm->payload_size); 91921e9f767SBen Widawsky return -ENXIO; 92021e9f767SBen Widawsky } 92121e9f767SBen Widawsky 92221e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, "Mailbox payload sized %zu", 92321e9f767SBen Widawsky cxlm->payload_size); 92421e9f767SBen Widawsky 92521e9f767SBen Widawsky return 0; 92621e9f767SBen Widawsky } 92721e9f767SBen Widawsky 92821e9f767SBen Widawsky static struct cxl_mem *cxl_mem_create(struct pci_dev *pdev, u32 reg_lo, 92921e9f767SBen Widawsky u32 reg_hi) 93021e9f767SBen Widawsky { 93121e9f767SBen Widawsky struct device *dev = &pdev->dev; 93221e9f767SBen Widawsky struct cxl_mem *cxlm; 93321e9f767SBen Widawsky void __iomem *regs; 93421e9f767SBen Widawsky u64 offset; 93521e9f767SBen Widawsky u8 bar; 93621e9f767SBen Widawsky int rc; 93721e9f767SBen Widawsky 938*5d0c6f02SBen Widawsky cxlm = devm_kzalloc(dev, sizeof(*cxlm), GFP_KERNEL); 93921e9f767SBen Widawsky if (!cxlm) { 94021e9f767SBen Widawsky dev_err(dev, "No memory available\n"); 94121e9f767SBen Widawsky return NULL; 94221e9f767SBen Widawsky } 94321e9f767SBen Widawsky 94421e9f767SBen Widawsky offset = ((u64)reg_hi << 32) | (reg_lo & CXL_REGLOC_ADDR_MASK); 94521e9f767SBen Widawsky bar = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo); 94621e9f767SBen Widawsky 94721e9f767SBen Widawsky /* Basic sanity check that BAR is big enough */ 94821e9f767SBen Widawsky if (pci_resource_len(pdev, bar) < offset) { 94921e9f767SBen Widawsky dev_err(dev, "BAR%d: %pr: too small (offset: %#llx)\n", bar, 95021e9f767SBen Widawsky &pdev->resource[bar], (unsigned long long)offset); 95121e9f767SBen Widawsky return NULL; 95221e9f767SBen Widawsky } 95321e9f767SBen Widawsky 95421e9f767SBen Widawsky rc = pcim_iomap_regions(pdev, BIT(bar), pci_name(pdev)); 95521e9f767SBen Widawsky if (rc) { 95621e9f767SBen Widawsky dev_err(dev, "failed to map registers\n"); 95721e9f767SBen Widawsky return NULL; 95821e9f767SBen Widawsky } 95921e9f767SBen Widawsky regs = pcim_iomap_table(pdev)[bar]; 96021e9f767SBen Widawsky 96121e9f767SBen Widawsky mutex_init(&cxlm->mbox_mutex); 96221e9f767SBen Widawsky cxlm->pdev = pdev; 96321e9f767SBen Widawsky cxlm->base = regs + offset; 96421e9f767SBen Widawsky cxlm->enabled_cmds = 96521e9f767SBen Widawsky devm_kmalloc_array(dev, BITS_TO_LONGS(cxl_cmd_count), 96621e9f767SBen Widawsky sizeof(unsigned long), 96721e9f767SBen Widawsky GFP_KERNEL | __GFP_ZERO); 96821e9f767SBen Widawsky if (!cxlm->enabled_cmds) { 96921e9f767SBen Widawsky dev_err(dev, "No memory available for bitmap\n"); 97021e9f767SBen Widawsky return NULL; 97121e9f767SBen Widawsky } 97221e9f767SBen Widawsky 97321e9f767SBen Widawsky dev_dbg(dev, "Mapped CXL Memory Device resource\n"); 97421e9f767SBen Widawsky return cxlm; 97521e9f767SBen Widawsky } 97621e9f767SBen Widawsky 97721e9f767SBen Widawsky static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec) 97821e9f767SBen Widawsky { 97921e9f767SBen Widawsky int pos; 98021e9f767SBen Widawsky 98121e9f767SBen Widawsky pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DVSEC); 98221e9f767SBen Widawsky if (!pos) 98321e9f767SBen Widawsky return 0; 98421e9f767SBen Widawsky 98521e9f767SBen Widawsky while (pos) { 98621e9f767SBen Widawsky u16 vendor, id; 98721e9f767SBen Widawsky 98821e9f767SBen Widawsky pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER1, &vendor); 98921e9f767SBen Widawsky pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER2, &id); 99021e9f767SBen Widawsky if (vendor == PCI_DVSEC_VENDOR_ID_CXL && dvsec == id) 99121e9f767SBen Widawsky return pos; 99221e9f767SBen Widawsky 99321e9f767SBen Widawsky pos = pci_find_next_ext_capability(pdev, pos, 99421e9f767SBen Widawsky PCI_EXT_CAP_ID_DVSEC); 99521e9f767SBen Widawsky } 99621e9f767SBen Widawsky 99721e9f767SBen Widawsky return 0; 99821e9f767SBen Widawsky } 99921e9f767SBen Widawsky 100021e9f767SBen Widawsky static struct cxl_memdev *to_cxl_memdev(struct device *dev) 100121e9f767SBen Widawsky { 100221e9f767SBen Widawsky return container_of(dev, struct cxl_memdev, dev); 100321e9f767SBen Widawsky } 100421e9f767SBen Widawsky 100521e9f767SBen Widawsky static void cxl_memdev_release(struct device *dev) 100621e9f767SBen Widawsky { 100721e9f767SBen Widawsky struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 100821e9f767SBen Widawsky 100921e9f767SBen Widawsky ida_free(&cxl_memdev_ida, cxlmd->id); 101021e9f767SBen Widawsky kfree(cxlmd); 101121e9f767SBen Widawsky } 101221e9f767SBen Widawsky 101321e9f767SBen Widawsky static char *cxl_memdev_devnode(struct device *dev, umode_t *mode, kuid_t *uid, 101421e9f767SBen Widawsky kgid_t *gid) 101521e9f767SBen Widawsky { 101621e9f767SBen Widawsky return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); 101721e9f767SBen Widawsky } 101821e9f767SBen Widawsky 101921e9f767SBen Widawsky static ssize_t firmware_version_show(struct device *dev, 102021e9f767SBen Widawsky struct device_attribute *attr, char *buf) 102121e9f767SBen Widawsky { 102221e9f767SBen Widawsky struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 102321e9f767SBen Widawsky struct cxl_mem *cxlm = cxlmd->cxlm; 102421e9f767SBen Widawsky 102521e9f767SBen Widawsky return sysfs_emit(buf, "%.16s\n", cxlm->firmware_version); 102621e9f767SBen Widawsky } 102721e9f767SBen Widawsky static DEVICE_ATTR_RO(firmware_version); 102821e9f767SBen Widawsky 102921e9f767SBen Widawsky static ssize_t payload_max_show(struct device *dev, 103021e9f767SBen Widawsky struct device_attribute *attr, char *buf) 103121e9f767SBen Widawsky { 103221e9f767SBen Widawsky struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 103321e9f767SBen Widawsky struct cxl_mem *cxlm = cxlmd->cxlm; 103421e9f767SBen Widawsky 103521e9f767SBen Widawsky return sysfs_emit(buf, "%zu\n", cxlm->payload_size); 103621e9f767SBen Widawsky } 103721e9f767SBen Widawsky static DEVICE_ATTR_RO(payload_max); 103821e9f767SBen Widawsky 1039199cf8c3SVishal Verma static ssize_t label_storage_size_show(struct device *dev, 1040199cf8c3SVishal Verma struct device_attribute *attr, char *buf) 1041199cf8c3SVishal Verma { 1042199cf8c3SVishal Verma struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 1043199cf8c3SVishal Verma struct cxl_mem *cxlm = cxlmd->cxlm; 1044199cf8c3SVishal Verma 1045199cf8c3SVishal Verma return sysfs_emit(buf, "%zu\n", cxlm->lsa_size); 1046199cf8c3SVishal Verma } 1047199cf8c3SVishal Verma static DEVICE_ATTR_RO(label_storage_size); 1048199cf8c3SVishal Verma 104921e9f767SBen Widawsky static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, 105021e9f767SBen Widawsky char *buf) 105121e9f767SBen Widawsky { 105221e9f767SBen Widawsky struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 105321e9f767SBen Widawsky struct cxl_mem *cxlm = cxlmd->cxlm; 105421e9f767SBen Widawsky unsigned long long len = range_len(&cxlm->ram_range); 105521e9f767SBen Widawsky 105621e9f767SBen Widawsky return sysfs_emit(buf, "%#llx\n", len); 105721e9f767SBen Widawsky } 105821e9f767SBen Widawsky 105921e9f767SBen Widawsky static struct device_attribute dev_attr_ram_size = 106021e9f767SBen Widawsky __ATTR(size, 0444, ram_size_show, NULL); 106121e9f767SBen Widawsky 106221e9f767SBen Widawsky static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, 106321e9f767SBen Widawsky char *buf) 106421e9f767SBen Widawsky { 106521e9f767SBen Widawsky struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 106621e9f767SBen Widawsky struct cxl_mem *cxlm = cxlmd->cxlm; 106721e9f767SBen Widawsky unsigned long long len = range_len(&cxlm->pmem_range); 106821e9f767SBen Widawsky 106921e9f767SBen Widawsky return sysfs_emit(buf, "%#llx\n", len); 107021e9f767SBen Widawsky } 107121e9f767SBen Widawsky 107221e9f767SBen Widawsky static struct device_attribute dev_attr_pmem_size = 107321e9f767SBen Widawsky __ATTR(size, 0444, pmem_size_show, NULL); 107421e9f767SBen Widawsky 107521e9f767SBen Widawsky static struct attribute *cxl_memdev_attributes[] = { 107621e9f767SBen Widawsky &dev_attr_firmware_version.attr, 107721e9f767SBen Widawsky &dev_attr_payload_max.attr, 1078199cf8c3SVishal Verma &dev_attr_label_storage_size.attr, 107921e9f767SBen Widawsky NULL, 108021e9f767SBen Widawsky }; 108121e9f767SBen Widawsky 108221e9f767SBen Widawsky static struct attribute *cxl_memdev_pmem_attributes[] = { 108321e9f767SBen Widawsky &dev_attr_pmem_size.attr, 108421e9f767SBen Widawsky NULL, 108521e9f767SBen Widawsky }; 108621e9f767SBen Widawsky 108721e9f767SBen Widawsky static struct attribute *cxl_memdev_ram_attributes[] = { 108821e9f767SBen Widawsky &dev_attr_ram_size.attr, 108921e9f767SBen Widawsky NULL, 109021e9f767SBen Widawsky }; 109121e9f767SBen Widawsky 109221e9f767SBen Widawsky static struct attribute_group cxl_memdev_attribute_group = { 109321e9f767SBen Widawsky .attrs = cxl_memdev_attributes, 109421e9f767SBen Widawsky }; 109521e9f767SBen Widawsky 109621e9f767SBen Widawsky static struct attribute_group cxl_memdev_ram_attribute_group = { 109721e9f767SBen Widawsky .name = "ram", 109821e9f767SBen Widawsky .attrs = cxl_memdev_ram_attributes, 109921e9f767SBen Widawsky }; 110021e9f767SBen Widawsky 110121e9f767SBen Widawsky static struct attribute_group cxl_memdev_pmem_attribute_group = { 110221e9f767SBen Widawsky .name = "pmem", 110321e9f767SBen Widawsky .attrs = cxl_memdev_pmem_attributes, 110421e9f767SBen Widawsky }; 110521e9f767SBen Widawsky 110621e9f767SBen Widawsky static const struct attribute_group *cxl_memdev_attribute_groups[] = { 110721e9f767SBen Widawsky &cxl_memdev_attribute_group, 110821e9f767SBen Widawsky &cxl_memdev_ram_attribute_group, 110921e9f767SBen Widawsky &cxl_memdev_pmem_attribute_group, 111021e9f767SBen Widawsky NULL, 111121e9f767SBen Widawsky }; 111221e9f767SBen Widawsky 111321e9f767SBen Widawsky static const struct device_type cxl_memdev_type = { 111421e9f767SBen Widawsky .name = "cxl_memdev", 111521e9f767SBen Widawsky .release = cxl_memdev_release, 111621e9f767SBen Widawsky .devnode = cxl_memdev_devnode, 111721e9f767SBen Widawsky .groups = cxl_memdev_attribute_groups, 111821e9f767SBen Widawsky }; 111921e9f767SBen Widawsky 112021e9f767SBen Widawsky static void cxl_memdev_shutdown(struct cxl_memdev *cxlmd) 112121e9f767SBen Widawsky { 112221e9f767SBen Widawsky down_write(&cxl_memdev_rwsem); 112321e9f767SBen Widawsky cxlmd->cxlm = NULL; 112421e9f767SBen Widawsky up_write(&cxl_memdev_rwsem); 112521e9f767SBen Widawsky } 112621e9f767SBen Widawsky 112721e9f767SBen Widawsky static void cxl_memdev_unregister(void *_cxlmd) 112821e9f767SBen Widawsky { 112921e9f767SBen Widawsky struct cxl_memdev *cxlmd = _cxlmd; 113021e9f767SBen Widawsky struct device *dev = &cxlmd->dev; 113121e9f767SBen Widawsky 113221e9f767SBen Widawsky cdev_device_del(&cxlmd->cdev, dev); 113321e9f767SBen Widawsky cxl_memdev_shutdown(cxlmd); 113421e9f767SBen Widawsky put_device(dev); 113521e9f767SBen Widawsky } 113621e9f767SBen Widawsky 113721e9f767SBen Widawsky static struct cxl_memdev *cxl_memdev_alloc(struct cxl_mem *cxlm) 113821e9f767SBen Widawsky { 113921e9f767SBen Widawsky struct pci_dev *pdev = cxlm->pdev; 114021e9f767SBen Widawsky struct cxl_memdev *cxlmd; 114121e9f767SBen Widawsky struct device *dev; 114221e9f767SBen Widawsky struct cdev *cdev; 114321e9f767SBen Widawsky int rc; 114421e9f767SBen Widawsky 114521e9f767SBen Widawsky cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL); 114621e9f767SBen Widawsky if (!cxlmd) 114721e9f767SBen Widawsky return ERR_PTR(-ENOMEM); 114821e9f767SBen Widawsky 114921e9f767SBen Widawsky rc = ida_alloc_range(&cxl_memdev_ida, 0, CXL_MEM_MAX_DEVS, GFP_KERNEL); 115021e9f767SBen Widawsky if (rc < 0) 115121e9f767SBen Widawsky goto err; 115221e9f767SBen Widawsky cxlmd->id = rc; 115321e9f767SBen Widawsky 115421e9f767SBen Widawsky dev = &cxlmd->dev; 115521e9f767SBen Widawsky device_initialize(dev); 115621e9f767SBen Widawsky dev->parent = &pdev->dev; 115721e9f767SBen Widawsky dev->bus = &cxl_bus_type; 115821e9f767SBen Widawsky dev->devt = MKDEV(cxl_mem_major, cxlmd->id); 115921e9f767SBen Widawsky dev->type = &cxl_memdev_type; 116021e9f767SBen Widawsky device_set_pm_not_required(dev); 116121e9f767SBen Widawsky 116221e9f767SBen Widawsky cdev = &cxlmd->cdev; 116321e9f767SBen Widawsky cdev_init(cdev, &cxl_memdev_fops); 116421e9f767SBen Widawsky return cxlmd; 116521e9f767SBen Widawsky 116621e9f767SBen Widawsky err: 116721e9f767SBen Widawsky kfree(cxlmd); 116821e9f767SBen Widawsky return ERR_PTR(rc); 116921e9f767SBen Widawsky } 117021e9f767SBen Widawsky 117121e9f767SBen Widawsky static int cxl_mem_add_memdev(struct cxl_mem *cxlm) 117221e9f767SBen Widawsky { 117321e9f767SBen Widawsky struct cxl_memdev *cxlmd; 117421e9f767SBen Widawsky struct device *dev; 117521e9f767SBen Widawsky struct cdev *cdev; 117621e9f767SBen Widawsky int rc; 117721e9f767SBen Widawsky 117821e9f767SBen Widawsky cxlmd = cxl_memdev_alloc(cxlm); 117921e9f767SBen Widawsky if (IS_ERR(cxlmd)) 118021e9f767SBen Widawsky return PTR_ERR(cxlmd); 118121e9f767SBen Widawsky 118221e9f767SBen Widawsky dev = &cxlmd->dev; 118321e9f767SBen Widawsky rc = dev_set_name(dev, "mem%d", cxlmd->id); 118421e9f767SBen Widawsky if (rc) 118521e9f767SBen Widawsky goto err; 118621e9f767SBen Widawsky 118721e9f767SBen Widawsky /* 118821e9f767SBen Widawsky * Activate ioctl operations, no cxl_memdev_rwsem manipulation 118921e9f767SBen Widawsky * needed as this is ordered with cdev_add() publishing the device. 119021e9f767SBen Widawsky */ 119121e9f767SBen Widawsky cxlmd->cxlm = cxlm; 119221e9f767SBen Widawsky 119321e9f767SBen Widawsky cdev = &cxlmd->cdev; 119421e9f767SBen Widawsky rc = cdev_device_add(cdev, dev); 119521e9f767SBen Widawsky if (rc) 119621e9f767SBen Widawsky goto err; 119721e9f767SBen Widawsky 119821e9f767SBen Widawsky return devm_add_action_or_reset(dev->parent, cxl_memdev_unregister, 119921e9f767SBen Widawsky cxlmd); 120021e9f767SBen Widawsky 120121e9f767SBen Widawsky err: 120221e9f767SBen Widawsky /* 120321e9f767SBen Widawsky * The cdev was briefly live, shutdown any ioctl operations that 120421e9f767SBen Widawsky * saw that state. 120521e9f767SBen Widawsky */ 120621e9f767SBen Widawsky cxl_memdev_shutdown(cxlmd); 120721e9f767SBen Widawsky put_device(dev); 120821e9f767SBen Widawsky return rc; 120921e9f767SBen Widawsky } 121021e9f767SBen Widawsky 121121e9f767SBen Widawsky static int cxl_xfer_log(struct cxl_mem *cxlm, uuid_t *uuid, u32 size, u8 *out) 121221e9f767SBen Widawsky { 121321e9f767SBen Widawsky u32 remaining = size; 121421e9f767SBen Widawsky u32 offset = 0; 121521e9f767SBen Widawsky 121621e9f767SBen Widawsky while (remaining) { 121721e9f767SBen Widawsky u32 xfer_size = min_t(u32, remaining, cxlm->payload_size); 121821e9f767SBen Widawsky struct cxl_mbox_get_log { 121921e9f767SBen Widawsky uuid_t uuid; 122021e9f767SBen Widawsky __le32 offset; 122121e9f767SBen Widawsky __le32 length; 122221e9f767SBen Widawsky } __packed log = { 122321e9f767SBen Widawsky .uuid = *uuid, 122421e9f767SBen Widawsky .offset = cpu_to_le32(offset), 122521e9f767SBen Widawsky .length = cpu_to_le32(xfer_size) 122621e9f767SBen Widawsky }; 122721e9f767SBen Widawsky int rc; 122821e9f767SBen Widawsky 122921e9f767SBen Widawsky rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_LOG, &log, 123021e9f767SBen Widawsky sizeof(log), out, xfer_size); 123121e9f767SBen Widawsky if (rc < 0) 123221e9f767SBen Widawsky return rc; 123321e9f767SBen Widawsky 123421e9f767SBen Widawsky out += xfer_size; 123521e9f767SBen Widawsky remaining -= xfer_size; 123621e9f767SBen Widawsky offset += xfer_size; 123721e9f767SBen Widawsky } 123821e9f767SBen Widawsky 123921e9f767SBen Widawsky return 0; 124021e9f767SBen Widawsky } 124121e9f767SBen Widawsky 124221e9f767SBen Widawsky /** 124321e9f767SBen Widawsky * cxl_walk_cel() - Walk through the Command Effects Log. 124421e9f767SBen Widawsky * @cxlm: Device. 124521e9f767SBen Widawsky * @size: Length of the Command Effects Log. 124621e9f767SBen Widawsky * @cel: CEL 124721e9f767SBen Widawsky * 124821e9f767SBen Widawsky * Iterate over each entry in the CEL and determine if the driver supports the 124921e9f767SBen Widawsky * command. If so, the command is enabled for the device and can be used later. 125021e9f767SBen Widawsky */ 125121e9f767SBen Widawsky static void cxl_walk_cel(struct cxl_mem *cxlm, size_t size, u8 *cel) 125221e9f767SBen Widawsky { 125321e9f767SBen Widawsky struct cel_entry { 125421e9f767SBen Widawsky __le16 opcode; 125521e9f767SBen Widawsky __le16 effect; 125621e9f767SBen Widawsky } __packed * cel_entry; 125721e9f767SBen Widawsky const int cel_entries = size / sizeof(*cel_entry); 125821e9f767SBen Widawsky int i; 125921e9f767SBen Widawsky 126021e9f767SBen Widawsky cel_entry = (struct cel_entry *)cel; 126121e9f767SBen Widawsky 126221e9f767SBen Widawsky for (i = 0; i < cel_entries; i++) { 126321e9f767SBen Widawsky u16 opcode = le16_to_cpu(cel_entry[i].opcode); 126421e9f767SBen Widawsky struct cxl_mem_command *cmd = cxl_mem_find_command(opcode); 126521e9f767SBen Widawsky 126621e9f767SBen Widawsky if (!cmd) { 126721e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, 126821e9f767SBen Widawsky "Opcode 0x%04x unsupported by driver", opcode); 126921e9f767SBen Widawsky continue; 127021e9f767SBen Widawsky } 127121e9f767SBen Widawsky 127221e9f767SBen Widawsky set_bit(cmd->info.id, cxlm->enabled_cmds); 127321e9f767SBen Widawsky } 127421e9f767SBen Widawsky } 127521e9f767SBen Widawsky 127621e9f767SBen Widawsky struct cxl_mbox_get_supported_logs { 127721e9f767SBen Widawsky __le16 entries; 127821e9f767SBen Widawsky u8 rsvd[6]; 127921e9f767SBen Widawsky struct gsl_entry { 128021e9f767SBen Widawsky uuid_t uuid; 128121e9f767SBen Widawsky __le32 size; 128221e9f767SBen Widawsky } __packed entry[]; 128321e9f767SBen Widawsky } __packed; 128421e9f767SBen Widawsky 128521e9f767SBen Widawsky static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_mem *cxlm) 128621e9f767SBen Widawsky { 128721e9f767SBen Widawsky struct cxl_mbox_get_supported_logs *ret; 128821e9f767SBen Widawsky int rc; 128921e9f767SBen Widawsky 129021e9f767SBen Widawsky ret = kvmalloc(cxlm->payload_size, GFP_KERNEL); 129121e9f767SBen Widawsky if (!ret) 129221e9f767SBen Widawsky return ERR_PTR(-ENOMEM); 129321e9f767SBen Widawsky 129421e9f767SBen Widawsky rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_SUPPORTED_LOGS, NULL, 129521e9f767SBen Widawsky 0, ret, cxlm->payload_size); 129621e9f767SBen Widawsky if (rc < 0) { 129721e9f767SBen Widawsky kvfree(ret); 129821e9f767SBen Widawsky return ERR_PTR(rc); 129921e9f767SBen Widawsky } 130021e9f767SBen Widawsky 130121e9f767SBen Widawsky return ret; 130221e9f767SBen Widawsky } 130321e9f767SBen Widawsky 130421e9f767SBen Widawsky /** 130521e9f767SBen Widawsky * cxl_mem_enumerate_cmds() - Enumerate commands for a device. 130621e9f767SBen Widawsky * @cxlm: The device. 130721e9f767SBen Widawsky * 130821e9f767SBen Widawsky * Returns 0 if enumerate completed successfully. 130921e9f767SBen Widawsky * 131021e9f767SBen Widawsky * CXL devices have optional support for certain commands. This function will 131121e9f767SBen Widawsky * determine the set of supported commands for the hardware and update the 131221e9f767SBen Widawsky * enabled_cmds bitmap in the @cxlm. 131321e9f767SBen Widawsky */ 131421e9f767SBen Widawsky static int cxl_mem_enumerate_cmds(struct cxl_mem *cxlm) 131521e9f767SBen Widawsky { 131621e9f767SBen Widawsky struct cxl_mbox_get_supported_logs *gsl; 131721e9f767SBen Widawsky struct device *dev = &cxlm->pdev->dev; 131821e9f767SBen Widawsky struct cxl_mem_command *cmd; 131921e9f767SBen Widawsky int i, rc; 132021e9f767SBen Widawsky 132121e9f767SBen Widawsky gsl = cxl_get_gsl(cxlm); 132221e9f767SBen Widawsky if (IS_ERR(gsl)) 132321e9f767SBen Widawsky return PTR_ERR(gsl); 132421e9f767SBen Widawsky 132521e9f767SBen Widawsky rc = -ENOENT; 132621e9f767SBen Widawsky for (i = 0; i < le16_to_cpu(gsl->entries); i++) { 132721e9f767SBen Widawsky u32 size = le32_to_cpu(gsl->entry[i].size); 132821e9f767SBen Widawsky uuid_t uuid = gsl->entry[i].uuid; 132921e9f767SBen Widawsky u8 *log; 133021e9f767SBen Widawsky 133121e9f767SBen Widawsky dev_dbg(dev, "Found LOG type %pU of size %d", &uuid, size); 133221e9f767SBen Widawsky 133321e9f767SBen Widawsky if (!uuid_equal(&uuid, &log_uuid[CEL_UUID])) 133421e9f767SBen Widawsky continue; 133521e9f767SBen Widawsky 133621e9f767SBen Widawsky log = kvmalloc(size, GFP_KERNEL); 133721e9f767SBen Widawsky if (!log) { 133821e9f767SBen Widawsky rc = -ENOMEM; 133921e9f767SBen Widawsky goto out; 134021e9f767SBen Widawsky } 134121e9f767SBen Widawsky 134221e9f767SBen Widawsky rc = cxl_xfer_log(cxlm, &uuid, size, log); 134321e9f767SBen Widawsky if (rc) { 134421e9f767SBen Widawsky kvfree(log); 134521e9f767SBen Widawsky goto out; 134621e9f767SBen Widawsky } 134721e9f767SBen Widawsky 134821e9f767SBen Widawsky cxl_walk_cel(cxlm, size, log); 134921e9f767SBen Widawsky kvfree(log); 135021e9f767SBen Widawsky 135121e9f767SBen Widawsky /* In case CEL was bogus, enable some default commands. */ 135221e9f767SBen Widawsky cxl_for_each_cmd(cmd) 135321e9f767SBen Widawsky if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE) 135421e9f767SBen Widawsky set_bit(cmd->info.id, cxlm->enabled_cmds); 135521e9f767SBen Widawsky 135621e9f767SBen Widawsky /* Found the required CEL */ 135721e9f767SBen Widawsky rc = 0; 135821e9f767SBen Widawsky } 135921e9f767SBen Widawsky 136021e9f767SBen Widawsky out: 136121e9f767SBen Widawsky kvfree(gsl); 136221e9f767SBen Widawsky return rc; 136321e9f767SBen Widawsky } 136421e9f767SBen Widawsky 136521e9f767SBen Widawsky /** 136621e9f767SBen Widawsky * cxl_mem_identify() - Send the IDENTIFY command to the device. 136721e9f767SBen Widawsky * @cxlm: The device to identify. 136821e9f767SBen Widawsky * 136921e9f767SBen Widawsky * Return: 0 if identify was executed successfully. 137021e9f767SBen Widawsky * 137121e9f767SBen Widawsky * This will dispatch the identify command to the device and on success populate 137221e9f767SBen Widawsky * structures to be exported to sysfs. 137321e9f767SBen Widawsky */ 137421e9f767SBen Widawsky static int cxl_mem_identify(struct cxl_mem *cxlm) 137521e9f767SBen Widawsky { 137621e9f767SBen Widawsky /* See CXL 2.0 Table 175 Identify Memory Device Output Payload */ 137721e9f767SBen Widawsky struct cxl_mbox_identify { 137821e9f767SBen Widawsky char fw_revision[0x10]; 137921e9f767SBen Widawsky __le64 total_capacity; 138021e9f767SBen Widawsky __le64 volatile_capacity; 138121e9f767SBen Widawsky __le64 persistent_capacity; 138221e9f767SBen Widawsky __le64 partition_align; 138321e9f767SBen Widawsky __le16 info_event_log_size; 138421e9f767SBen Widawsky __le16 warning_event_log_size; 138521e9f767SBen Widawsky __le16 failure_event_log_size; 138621e9f767SBen Widawsky __le16 fatal_event_log_size; 138721e9f767SBen Widawsky __le32 lsa_size; 138821e9f767SBen Widawsky u8 poison_list_max_mer[3]; 138921e9f767SBen Widawsky __le16 inject_poison_limit; 139021e9f767SBen Widawsky u8 poison_caps; 139121e9f767SBen Widawsky u8 qos_telemetry_caps; 139221e9f767SBen Widawsky } __packed id; 139321e9f767SBen Widawsky int rc; 139421e9f767SBen Widawsky 139521e9f767SBen Widawsky rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_IDENTIFY, NULL, 0, &id, 139621e9f767SBen Widawsky sizeof(id)); 139721e9f767SBen Widawsky if (rc < 0) 139821e9f767SBen Widawsky return rc; 139921e9f767SBen Widawsky 140021e9f767SBen Widawsky /* 140121e9f767SBen Widawsky * TODO: enumerate DPA map, as 'ram' and 'pmem' do not alias. 140221e9f767SBen Widawsky * For now, only the capacity is exported in sysfs 140321e9f767SBen Widawsky */ 140421e9f767SBen Widawsky cxlm->ram_range.start = 0; 140521e9f767SBen Widawsky cxlm->ram_range.end = le64_to_cpu(id.volatile_capacity) * SZ_256M - 1; 140621e9f767SBen Widawsky 140721e9f767SBen Widawsky cxlm->pmem_range.start = 0; 140821e9f767SBen Widawsky cxlm->pmem_range.end = 140921e9f767SBen Widawsky le64_to_cpu(id.persistent_capacity) * SZ_256M - 1; 141021e9f767SBen Widawsky 1411199cf8c3SVishal Verma cxlm->lsa_size = le32_to_cpu(id.lsa_size); 141221e9f767SBen Widawsky memcpy(cxlm->firmware_version, id.fw_revision, sizeof(id.fw_revision)); 141321e9f767SBen Widawsky 141421e9f767SBen Widawsky return 0; 141521e9f767SBen Widawsky } 141621e9f767SBen Widawsky 141721e9f767SBen Widawsky static int cxl_mem_probe(struct pci_dev *pdev, const struct pci_device_id *id) 141821e9f767SBen Widawsky { 141921e9f767SBen Widawsky struct device *dev = &pdev->dev; 142021e9f767SBen Widawsky struct cxl_mem *cxlm = NULL; 142121e9f767SBen Widawsky u32 regloc_size, regblocks; 142221e9f767SBen Widawsky int rc, regloc, i; 142321e9f767SBen Widawsky 142421e9f767SBen Widawsky rc = pcim_enable_device(pdev); 142521e9f767SBen Widawsky if (rc) 142621e9f767SBen Widawsky return rc; 142721e9f767SBen Widawsky 142821e9f767SBen Widawsky regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_OFFSET); 142921e9f767SBen Widawsky if (!regloc) { 143021e9f767SBen Widawsky dev_err(dev, "register location dvsec not found\n"); 143121e9f767SBen Widawsky return -ENXIO; 143221e9f767SBen Widawsky } 143321e9f767SBen Widawsky 143421e9f767SBen Widawsky /* Get the size of the Register Locator DVSEC */ 143521e9f767SBen Widawsky pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size); 143621e9f767SBen Widawsky regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size); 143721e9f767SBen Widawsky 143821e9f767SBen Widawsky regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET; 143921e9f767SBen Widawsky regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8; 144021e9f767SBen Widawsky 144121e9f767SBen Widawsky for (i = 0; i < regblocks; i++, regloc += 8) { 144221e9f767SBen Widawsky u32 reg_lo, reg_hi; 144321e9f767SBen Widawsky u8 reg_type; 144421e9f767SBen Widawsky 144521e9f767SBen Widawsky /* "register low and high" contain other bits */ 144621e9f767SBen Widawsky pci_read_config_dword(pdev, regloc, ®_lo); 144721e9f767SBen Widawsky pci_read_config_dword(pdev, regloc + 4, ®_hi); 144821e9f767SBen Widawsky 144921e9f767SBen Widawsky reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); 145021e9f767SBen Widawsky 145121e9f767SBen Widawsky if (reg_type == CXL_REGLOC_RBI_MEMDEV) { 145221e9f767SBen Widawsky cxlm = cxl_mem_create(pdev, reg_lo, reg_hi); 145321e9f767SBen Widawsky break; 145421e9f767SBen Widawsky } 145521e9f767SBen Widawsky } 145621e9f767SBen Widawsky 145721e9f767SBen Widawsky if (!cxlm) 145821e9f767SBen Widawsky return -ENODEV; 145921e9f767SBen Widawsky 146021e9f767SBen Widawsky rc = cxl_mem_setup_regs(cxlm); 146121e9f767SBen Widawsky if (rc) 146221e9f767SBen Widawsky return rc; 146321e9f767SBen Widawsky 146421e9f767SBen Widawsky rc = cxl_mem_setup_mailbox(cxlm); 146521e9f767SBen Widawsky if (rc) 146621e9f767SBen Widawsky return rc; 146721e9f767SBen Widawsky 146821e9f767SBen Widawsky rc = cxl_mem_enumerate_cmds(cxlm); 146921e9f767SBen Widawsky if (rc) 147021e9f767SBen Widawsky return rc; 147121e9f767SBen Widawsky 147221e9f767SBen Widawsky rc = cxl_mem_identify(cxlm); 147321e9f767SBen Widawsky if (rc) 147421e9f767SBen Widawsky return rc; 147521e9f767SBen Widawsky 147621e9f767SBen Widawsky return cxl_mem_add_memdev(cxlm); 147721e9f767SBen Widawsky } 147821e9f767SBen Widawsky 147921e9f767SBen Widawsky static const struct pci_device_id cxl_mem_pci_tbl[] = { 148021e9f767SBen Widawsky /* PCI class code for CXL.mem Type-3 Devices */ 148121e9f767SBen Widawsky { PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)}, 148221e9f767SBen Widawsky { /* terminate list */ }, 148321e9f767SBen Widawsky }; 148421e9f767SBen Widawsky MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl); 148521e9f767SBen Widawsky 148621e9f767SBen Widawsky static struct pci_driver cxl_mem_driver = { 148721e9f767SBen Widawsky .name = KBUILD_MODNAME, 148821e9f767SBen Widawsky .id_table = cxl_mem_pci_tbl, 148921e9f767SBen Widawsky .probe = cxl_mem_probe, 149021e9f767SBen Widawsky .driver = { 149121e9f767SBen Widawsky .probe_type = PROBE_PREFER_ASYNCHRONOUS, 149221e9f767SBen Widawsky }, 149321e9f767SBen Widawsky }; 149421e9f767SBen Widawsky 149521e9f767SBen Widawsky static __init int cxl_mem_init(void) 149621e9f767SBen Widawsky { 149721e9f767SBen Widawsky struct dentry *mbox_debugfs; 149821e9f767SBen Widawsky dev_t devt; 149921e9f767SBen Widawsky int rc; 150021e9f767SBen Widawsky 150121e9f767SBen Widawsky /* Double check the anonymous union trickery in struct cxl_regs */ 150221e9f767SBen Widawsky BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) != 150321e9f767SBen Widawsky offsetof(struct cxl_regs, device_regs.memdev)); 150421e9f767SBen Widawsky 150521e9f767SBen Widawsky rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); 150621e9f767SBen Widawsky if (rc) 150721e9f767SBen Widawsky return rc; 150821e9f767SBen Widawsky 150921e9f767SBen Widawsky cxl_mem_major = MAJOR(devt); 151021e9f767SBen Widawsky 151121e9f767SBen Widawsky rc = pci_register_driver(&cxl_mem_driver); 151221e9f767SBen Widawsky if (rc) { 151321e9f767SBen Widawsky unregister_chrdev_region(MKDEV(cxl_mem_major, 0), 151421e9f767SBen Widawsky CXL_MEM_MAX_DEVS); 151521e9f767SBen Widawsky return rc; 151621e9f767SBen Widawsky } 151721e9f767SBen Widawsky 151821e9f767SBen Widawsky cxl_debugfs = debugfs_create_dir("cxl", NULL); 151921e9f767SBen Widawsky mbox_debugfs = debugfs_create_dir("mbox", cxl_debugfs); 152021e9f767SBen Widawsky debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs, 152121e9f767SBen Widawsky &cxl_raw_allow_all); 152221e9f767SBen Widawsky 152321e9f767SBen Widawsky return 0; 152421e9f767SBen Widawsky } 152521e9f767SBen Widawsky 152621e9f767SBen Widawsky static __exit void cxl_mem_exit(void) 152721e9f767SBen Widawsky { 152821e9f767SBen Widawsky debugfs_remove_recursive(cxl_debugfs); 152921e9f767SBen Widawsky pci_unregister_driver(&cxl_mem_driver); 153021e9f767SBen Widawsky unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS); 153121e9f767SBen Widawsky } 153221e9f767SBen Widawsky 153321e9f767SBen Widawsky MODULE_LICENSE("GPL v2"); 153421e9f767SBen Widawsky module_init(cxl_mem_init); 153521e9f767SBen Widawsky module_exit(cxl_mem_exit); 153621e9f767SBen Widawsky MODULE_IMPORT_NS(CXL); 1537