121e9f767SBen Widawsky // SPDX-License-Identifier: GPL-2.0-only 221e9f767SBen Widawsky /* Copyright(c) 2020 Intel Corporation. All rights reserved. */ 321e9f767SBen Widawsky #include <uapi/linux/cxl_mem.h> 421e9f767SBen Widawsky #include <linux/security.h> 521e9f767SBen Widawsky #include <linux/debugfs.h> 621e9f767SBen Widawsky #include <linux/module.h> 721e9f767SBen Widawsky #include <linux/sizes.h> 821e9f767SBen Widawsky #include <linux/mutex.h> 930af9729SIra Weiny #include <linux/list.h> 1021e9f767SBen Widawsky #include <linux/cdev.h> 1121e9f767SBen Widawsky #include <linux/idr.h> 1221e9f767SBen Widawsky #include <linux/pci.h> 1321e9f767SBen Widawsky #include <linux/io.h> 1421e9f767SBen Widawsky #include <linux/io-64-nonatomic-lo-hi.h> 155161a55cSBen Widawsky #include "cxlmem.h" 1621e9f767SBen Widawsky #include "pci.h" 1721e9f767SBen Widawsky #include "cxl.h" 1821e9f767SBen Widawsky 1921e9f767SBen Widawsky /** 2021e9f767SBen Widawsky * DOC: cxl pci 2121e9f767SBen Widawsky * 2221e9f767SBen Widawsky * This implements the PCI exclusive functionality for a CXL device as it is 2321e9f767SBen Widawsky * defined by the Compute Express Link specification. CXL devices may surface 2421e9f767SBen Widawsky * certain functionality even if it isn't CXL enabled. 2521e9f767SBen Widawsky * 2621e9f767SBen Widawsky * The driver has several responsibilities, mainly: 2721e9f767SBen Widawsky * - Create the memX device and register on the CXL bus. 2821e9f767SBen Widawsky * - Enumerate device's register interface and map them. 2921e9f767SBen Widawsky * - Probe the device attributes to establish sysfs interface. 3021e9f767SBen Widawsky * - Provide an IOCTL interface to userspace to communicate with the device for 3121e9f767SBen Widawsky * things like firmware update. 3221e9f767SBen Widawsky */ 3321e9f767SBen Widawsky 3421e9f767SBen Widawsky #define cxl_doorbell_busy(cxlm) \ 3521e9f767SBen Widawsky (readl((cxlm)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \ 3621e9f767SBen Widawsky CXLDEV_MBOX_CTRL_DOORBELL) 3721e9f767SBen Widawsky 3821e9f767SBen Widawsky /* CXL 2.0 - 8.2.8.4 */ 3921e9f767SBen Widawsky #define CXL_MAILBOX_TIMEOUT_MS (2 * HZ) 4021e9f767SBen Widawsky 4121e9f767SBen Widawsky enum opcode { 4221e9f767SBen Widawsky CXL_MBOX_OP_INVALID = 0x0000, 4321e9f767SBen Widawsky CXL_MBOX_OP_RAW = CXL_MBOX_OP_INVALID, 4421e9f767SBen Widawsky CXL_MBOX_OP_GET_FW_INFO = 0x0200, 4521e9f767SBen Widawsky CXL_MBOX_OP_ACTIVATE_FW = 0x0202, 4621e9f767SBen Widawsky CXL_MBOX_OP_GET_SUPPORTED_LOGS = 0x0400, 4721e9f767SBen Widawsky CXL_MBOX_OP_GET_LOG = 0x0401, 4821e9f767SBen Widawsky CXL_MBOX_OP_IDENTIFY = 0x4000, 4921e9f767SBen Widawsky CXL_MBOX_OP_GET_PARTITION_INFO = 0x4100, 5021e9f767SBen Widawsky CXL_MBOX_OP_SET_PARTITION_INFO = 0x4101, 5121e9f767SBen Widawsky CXL_MBOX_OP_GET_LSA = 0x4102, 5221e9f767SBen Widawsky CXL_MBOX_OP_SET_LSA = 0x4103, 5321e9f767SBen Widawsky CXL_MBOX_OP_GET_HEALTH_INFO = 0x4200, 5487815ee9SBen Widawsky CXL_MBOX_OP_GET_ALERT_CONFIG = 0x4201, 5587815ee9SBen Widawsky CXL_MBOX_OP_SET_ALERT_CONFIG = 0x4202, 5687815ee9SBen Widawsky CXL_MBOX_OP_GET_SHUTDOWN_STATE = 0x4203, 5721e9f767SBen Widawsky CXL_MBOX_OP_SET_SHUTDOWN_STATE = 0x4204, 5887815ee9SBen Widawsky CXL_MBOX_OP_GET_POISON = 0x4300, 5987815ee9SBen Widawsky CXL_MBOX_OP_INJECT_POISON = 0x4301, 6087815ee9SBen Widawsky CXL_MBOX_OP_CLEAR_POISON = 0x4302, 6187815ee9SBen Widawsky CXL_MBOX_OP_GET_SCAN_MEDIA_CAPS = 0x4303, 6221e9f767SBen Widawsky CXL_MBOX_OP_SCAN_MEDIA = 0x4304, 6321e9f767SBen Widawsky CXL_MBOX_OP_GET_SCAN_MEDIA = 0x4305, 6421e9f767SBen Widawsky CXL_MBOX_OP_MAX = 0x10000 6521e9f767SBen Widawsky }; 6621e9f767SBen Widawsky 670b9159d0SIra Weiny /* 680b9159d0SIra Weiny * CXL 2.0 - Memory capacity multiplier 690b9159d0SIra Weiny * See Section 8.2.9.5 700b9159d0SIra Weiny * 710b9159d0SIra Weiny * Volatile, Persistent, and Partition capacities are specified to be in 720b9159d0SIra Weiny * multiples of 256MB - define a multiplier to convert to/from bytes. 730b9159d0SIra Weiny */ 740b9159d0SIra Weiny #define CXL_CAPACITY_MULTIPLIER SZ_256M 750b9159d0SIra Weiny 7621e9f767SBen Widawsky /** 7721e9f767SBen Widawsky * struct mbox_cmd - A command to be submitted to hardware. 7821e9f767SBen Widawsky * @opcode: (input) The command set and command submitted to hardware. 7921e9f767SBen Widawsky * @payload_in: (input) Pointer to the input payload. 8021e9f767SBen Widawsky * @payload_out: (output) Pointer to the output payload. Must be allocated by 8121e9f767SBen Widawsky * the caller. 8221e9f767SBen Widawsky * @size_in: (input) Number of bytes to load from @payload_in. 8321e9f767SBen Widawsky * @size_out: (input) Max number of bytes loaded into @payload_out. 8421e9f767SBen Widawsky * (output) Number of bytes generated by the device. For fixed size 8521e9f767SBen Widawsky * outputs commands this is always expected to be deterministic. For 8621e9f767SBen Widawsky * variable sized output commands, it tells the exact number of bytes 8721e9f767SBen Widawsky * written. 8821e9f767SBen Widawsky * @return_code: (output) Error code returned from hardware. 8921e9f767SBen Widawsky * 9021e9f767SBen Widawsky * This is the primary mechanism used to send commands to the hardware. 9121e9f767SBen Widawsky * All the fields except @payload_* correspond exactly to the fields described in 9221e9f767SBen Widawsky * Command Register section of the CXL 2.0 8.2.8.4.5. @payload_in and 9321e9f767SBen Widawsky * @payload_out are written to, and read from the Command Payload Registers 9421e9f767SBen Widawsky * defined in CXL 2.0 8.2.8.4.8. 9521e9f767SBen Widawsky */ 9621e9f767SBen Widawsky struct mbox_cmd { 9721e9f767SBen Widawsky u16 opcode; 9821e9f767SBen Widawsky void *payload_in; 9921e9f767SBen Widawsky void *payload_out; 10021e9f767SBen Widawsky size_t size_in; 10121e9f767SBen Widawsky size_t size_out; 10221e9f767SBen Widawsky u16 return_code; 10321e9f767SBen Widawsky #define CXL_MBOX_SUCCESS 0 10421e9f767SBen Widawsky }; 10521e9f767SBen Widawsky 10621e9f767SBen Widawsky static DECLARE_RWSEM(cxl_memdev_rwsem); 10721e9f767SBen Widawsky static struct dentry *cxl_debugfs; 10821e9f767SBen Widawsky static bool cxl_raw_allow_all; 10921e9f767SBen Widawsky 11021e9f767SBen Widawsky enum { 11121e9f767SBen Widawsky CEL_UUID, 11221e9f767SBen Widawsky VENDOR_DEBUG_UUID, 11321e9f767SBen Widawsky }; 11421e9f767SBen Widawsky 11521e9f767SBen Widawsky /* See CXL 2.0 Table 170. Get Log Input Payload */ 11621e9f767SBen Widawsky static const uuid_t log_uuid[] = { 11721e9f767SBen Widawsky [CEL_UUID] = UUID_INIT(0xda9c0b5, 0xbf41, 0x4b78, 0x8f, 0x79, 0x96, 11821e9f767SBen Widawsky 0xb1, 0x62, 0x3b, 0x3f, 0x17), 11921e9f767SBen Widawsky [VENDOR_DEBUG_UUID] = UUID_INIT(0xe1819d9, 0x11a9, 0x400c, 0x81, 0x1f, 12021e9f767SBen Widawsky 0xd6, 0x07, 0x19, 0x40, 0x3d, 0x86), 12121e9f767SBen Widawsky }; 12221e9f767SBen Widawsky 12321e9f767SBen Widawsky /** 12421e9f767SBen Widawsky * struct cxl_mem_command - Driver representation of a memory device command 12521e9f767SBen Widawsky * @info: Command information as it exists for the UAPI 12621e9f767SBen Widawsky * @opcode: The actual bits used for the mailbox protocol 12721e9f767SBen Widawsky * @flags: Set of flags effecting driver behavior. 12821e9f767SBen Widawsky * 12921e9f767SBen Widawsky * * %CXL_CMD_FLAG_FORCE_ENABLE: In cases of error, commands with this flag 13021e9f767SBen Widawsky * will be enabled by the driver regardless of what hardware may have 13121e9f767SBen Widawsky * advertised. 13221e9f767SBen Widawsky * 13321e9f767SBen Widawsky * The cxl_mem_command is the driver's internal representation of commands that 13421e9f767SBen Widawsky * are supported by the driver. Some of these commands may not be supported by 13521e9f767SBen Widawsky * the hardware. The driver will use @info to validate the fields passed in by 13621e9f767SBen Widawsky * the user then submit the @opcode to the hardware. 13721e9f767SBen Widawsky * 13821e9f767SBen Widawsky * See struct cxl_command_info. 13921e9f767SBen Widawsky */ 14021e9f767SBen Widawsky struct cxl_mem_command { 14121e9f767SBen Widawsky struct cxl_command_info info; 14221e9f767SBen Widawsky enum opcode opcode; 14321e9f767SBen Widawsky u32 flags; 14421e9f767SBen Widawsky #define CXL_CMD_FLAG_NONE 0 14521e9f767SBen Widawsky #define CXL_CMD_FLAG_FORCE_ENABLE BIT(0) 14621e9f767SBen Widawsky }; 14721e9f767SBen Widawsky 14821e9f767SBen Widawsky #define CXL_CMD(_id, sin, sout, _flags) \ 14921e9f767SBen Widawsky [CXL_MEM_COMMAND_ID_##_id] = { \ 15021e9f767SBen Widawsky .info = { \ 15121e9f767SBen Widawsky .id = CXL_MEM_COMMAND_ID_##_id, \ 15221e9f767SBen Widawsky .size_in = sin, \ 15321e9f767SBen Widawsky .size_out = sout, \ 15421e9f767SBen Widawsky }, \ 15521e9f767SBen Widawsky .opcode = CXL_MBOX_OP_##_id, \ 15621e9f767SBen Widawsky .flags = _flags, \ 15721e9f767SBen Widawsky } 15821e9f767SBen Widawsky 15921e9f767SBen Widawsky /* 16021e9f767SBen Widawsky * This table defines the supported mailbox commands for the driver. This table 16121e9f767SBen Widawsky * is made up of a UAPI structure. Non-negative values as parameters in the 16221e9f767SBen Widawsky * table will be validated against the user's input. For example, if size_in is 16321e9f767SBen Widawsky * 0, and the user passed in 1, it is an error. 16421e9f767SBen Widawsky */ 16521e9f767SBen Widawsky static struct cxl_mem_command mem_commands[CXL_MEM_COMMAND_ID_MAX] = { 16621e9f767SBen Widawsky CXL_CMD(IDENTIFY, 0, 0x43, CXL_CMD_FLAG_FORCE_ENABLE), 16721e9f767SBen Widawsky #ifdef CONFIG_CXL_MEM_RAW_COMMANDS 16821e9f767SBen Widawsky CXL_CMD(RAW, ~0, ~0, 0), 16921e9f767SBen Widawsky #endif 17021e9f767SBen Widawsky CXL_CMD(GET_SUPPORTED_LOGS, 0, ~0, CXL_CMD_FLAG_FORCE_ENABLE), 17121e9f767SBen Widawsky CXL_CMD(GET_FW_INFO, 0, 0x50, 0), 17221e9f767SBen Widawsky CXL_CMD(GET_PARTITION_INFO, 0, 0x20, 0), 17321e9f767SBen Widawsky CXL_CMD(GET_LSA, 0x8, ~0, 0), 17421e9f767SBen Widawsky CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0), 17521e9f767SBen Widawsky CXL_CMD(GET_LOG, 0x18, ~0, CXL_CMD_FLAG_FORCE_ENABLE), 17687815ee9SBen Widawsky CXL_CMD(SET_PARTITION_INFO, 0x0a, 0, 0), 17787815ee9SBen Widawsky CXL_CMD(SET_LSA, ~0, 0, 0), 17887815ee9SBen Widawsky CXL_CMD(GET_ALERT_CONFIG, 0, 0x10, 0), 17987815ee9SBen Widawsky CXL_CMD(SET_ALERT_CONFIG, 0xc, 0, 0), 18087815ee9SBen Widawsky CXL_CMD(GET_SHUTDOWN_STATE, 0, 0x1, 0), 18187815ee9SBen Widawsky CXL_CMD(SET_SHUTDOWN_STATE, 0x1, 0, 0), 18287815ee9SBen Widawsky CXL_CMD(GET_POISON, 0x10, ~0, 0), 18387815ee9SBen Widawsky CXL_CMD(INJECT_POISON, 0x8, 0, 0), 18487815ee9SBen Widawsky CXL_CMD(CLEAR_POISON, 0x48, 0, 0), 18587815ee9SBen Widawsky CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0), 18687815ee9SBen Widawsky CXL_CMD(SCAN_MEDIA, 0x11, 0, 0), 18787815ee9SBen Widawsky CXL_CMD(GET_SCAN_MEDIA, 0, ~0, 0), 18821e9f767SBen Widawsky }; 18921e9f767SBen Widawsky 19021e9f767SBen Widawsky /* 19121e9f767SBen Widawsky * Commands that RAW doesn't permit. The rationale for each: 19221e9f767SBen Widawsky * 19321e9f767SBen Widawsky * CXL_MBOX_OP_ACTIVATE_FW: Firmware activation requires adjustment / 19421e9f767SBen Widawsky * coordination of transaction timeout values at the root bridge level. 19521e9f767SBen Widawsky * 19621e9f767SBen Widawsky * CXL_MBOX_OP_SET_PARTITION_INFO: The device memory map may change live 19721e9f767SBen Widawsky * and needs to be coordinated with HDM updates. 19821e9f767SBen Widawsky * 19921e9f767SBen Widawsky * CXL_MBOX_OP_SET_LSA: The label storage area may be cached by the 20021e9f767SBen Widawsky * driver and any writes from userspace invalidates those contents. 20121e9f767SBen Widawsky * 20221e9f767SBen Widawsky * CXL_MBOX_OP_SET_SHUTDOWN_STATE: Set shutdown state assumes no writes 20321e9f767SBen Widawsky * to the device after it is marked clean, userspace can not make that 20421e9f767SBen Widawsky * assertion. 20521e9f767SBen Widawsky * 20621e9f767SBen Widawsky * CXL_MBOX_OP_[GET_]SCAN_MEDIA: The kernel provides a native error list that 20721e9f767SBen Widawsky * is kept up to date with patrol notifications and error management. 20821e9f767SBen Widawsky */ 20921e9f767SBen Widawsky static u16 cxl_disabled_raw_commands[] = { 21021e9f767SBen Widawsky CXL_MBOX_OP_ACTIVATE_FW, 21121e9f767SBen Widawsky CXL_MBOX_OP_SET_PARTITION_INFO, 21221e9f767SBen Widawsky CXL_MBOX_OP_SET_LSA, 21321e9f767SBen Widawsky CXL_MBOX_OP_SET_SHUTDOWN_STATE, 21421e9f767SBen Widawsky CXL_MBOX_OP_SCAN_MEDIA, 21521e9f767SBen Widawsky CXL_MBOX_OP_GET_SCAN_MEDIA, 21621e9f767SBen Widawsky }; 21721e9f767SBen Widawsky 21821e9f767SBen Widawsky /* 21921e9f767SBen Widawsky * Command sets that RAW doesn't permit. All opcodes in this set are 22021e9f767SBen Widawsky * disabled because they pass plain text security payloads over the 22121e9f767SBen Widawsky * user/kernel boundary. This functionality is intended to be wrapped 22221e9f767SBen Widawsky * behind the keys ABI which allows for encrypted payloads in the UAPI 22321e9f767SBen Widawsky */ 22421e9f767SBen Widawsky static u8 security_command_sets[] = { 22521e9f767SBen Widawsky 0x44, /* Sanitize */ 22621e9f767SBen Widawsky 0x45, /* Persistent Memory Data-at-rest Security */ 22721e9f767SBen Widawsky 0x46, /* Security Passthrough */ 22821e9f767SBen Widawsky }; 22921e9f767SBen Widawsky 23021e9f767SBen Widawsky #define cxl_for_each_cmd(cmd) \ 23121e9f767SBen Widawsky for ((cmd) = &mem_commands[0]; \ 23221e9f767SBen Widawsky ((cmd) - mem_commands) < ARRAY_SIZE(mem_commands); (cmd)++) 23321e9f767SBen Widawsky 23421e9f767SBen Widawsky #define cxl_cmd_count ARRAY_SIZE(mem_commands) 23521e9f767SBen Widawsky 23621e9f767SBen Widawsky static int cxl_mem_wait_for_doorbell(struct cxl_mem *cxlm) 23721e9f767SBen Widawsky { 23821e9f767SBen Widawsky const unsigned long start = jiffies; 23921e9f767SBen Widawsky unsigned long end = start; 24021e9f767SBen Widawsky 24121e9f767SBen Widawsky while (cxl_doorbell_busy(cxlm)) { 24221e9f767SBen Widawsky end = jiffies; 24321e9f767SBen Widawsky 24421e9f767SBen Widawsky if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) { 24521e9f767SBen Widawsky /* Check again in case preempted before timeout test */ 24621e9f767SBen Widawsky if (!cxl_doorbell_busy(cxlm)) 24721e9f767SBen Widawsky break; 24821e9f767SBen Widawsky return -ETIMEDOUT; 24921e9f767SBen Widawsky } 25021e9f767SBen Widawsky cpu_relax(); 25121e9f767SBen Widawsky } 25221e9f767SBen Widawsky 25321e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, "Doorbell wait took %dms", 25421e9f767SBen Widawsky jiffies_to_msecs(end) - jiffies_to_msecs(start)); 25521e9f767SBen Widawsky return 0; 25621e9f767SBen Widawsky } 25721e9f767SBen Widawsky 25821e9f767SBen Widawsky static bool cxl_is_security_command(u16 opcode) 25921e9f767SBen Widawsky { 26021e9f767SBen Widawsky int i; 26121e9f767SBen Widawsky 26221e9f767SBen Widawsky for (i = 0; i < ARRAY_SIZE(security_command_sets); i++) 26321e9f767SBen Widawsky if (security_command_sets[i] == (opcode >> 8)) 26421e9f767SBen Widawsky return true; 26521e9f767SBen Widawsky return false; 26621e9f767SBen Widawsky } 26721e9f767SBen Widawsky 26821e9f767SBen Widawsky static void cxl_mem_mbox_timeout(struct cxl_mem *cxlm, 26921e9f767SBen Widawsky struct mbox_cmd *mbox_cmd) 27021e9f767SBen Widawsky { 27121e9f767SBen Widawsky struct device *dev = &cxlm->pdev->dev; 27221e9f767SBen Widawsky 27321e9f767SBen Widawsky dev_dbg(dev, "Mailbox command (opcode: %#x size: %zub) timed out\n", 27421e9f767SBen Widawsky mbox_cmd->opcode, mbox_cmd->size_in); 27521e9f767SBen Widawsky } 27621e9f767SBen Widawsky 27721e9f767SBen Widawsky /** 27821e9f767SBen Widawsky * __cxl_mem_mbox_send_cmd() - Execute a mailbox command 27921e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 28021e9f767SBen Widawsky * @mbox_cmd: Command to send to the memory device. 28121e9f767SBen Widawsky * 28221e9f767SBen Widawsky * Context: Any context. Expects mbox_mutex to be held. 28321e9f767SBen Widawsky * Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success. 28421e9f767SBen Widawsky * Caller should check the return code in @mbox_cmd to make sure it 28521e9f767SBen Widawsky * succeeded. 28621e9f767SBen Widawsky * 28721e9f767SBen Widawsky * This is a generic form of the CXL mailbox send command thus only using the 28821e9f767SBen Widawsky * registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory 28921e9f767SBen Widawsky * devices, and perhaps other types of CXL devices may have further information 29021e9f767SBen Widawsky * available upon error conditions. Driver facilities wishing to send mailbox 29121e9f767SBen Widawsky * commands should use the wrapper command. 29221e9f767SBen Widawsky * 29321e9f767SBen Widawsky * The CXL spec allows for up to two mailboxes. The intention is for the primary 29421e9f767SBen Widawsky * mailbox to be OS controlled and the secondary mailbox to be used by system 29521e9f767SBen Widawsky * firmware. This allows the OS and firmware to communicate with the device and 29621e9f767SBen Widawsky * not need to coordinate with each other. The driver only uses the primary 29721e9f767SBen Widawsky * mailbox. 29821e9f767SBen Widawsky */ 29921e9f767SBen Widawsky static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, 30021e9f767SBen Widawsky struct mbox_cmd *mbox_cmd) 30121e9f767SBen Widawsky { 30221e9f767SBen Widawsky void __iomem *payload = cxlm->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET; 30321e9f767SBen Widawsky u64 cmd_reg, status_reg; 30421e9f767SBen Widawsky size_t out_len; 30521e9f767SBen Widawsky int rc; 30621e9f767SBen Widawsky 30721e9f767SBen Widawsky lockdep_assert_held(&cxlm->mbox_mutex); 30821e9f767SBen Widawsky 30921e9f767SBen Widawsky /* 31021e9f767SBen Widawsky * Here are the steps from 8.2.8.4 of the CXL 2.0 spec. 31121e9f767SBen Widawsky * 1. Caller reads MB Control Register to verify doorbell is clear 31221e9f767SBen Widawsky * 2. Caller writes Command Register 31321e9f767SBen Widawsky * 3. Caller writes Command Payload Registers if input payload is non-empty 31421e9f767SBen Widawsky * 4. Caller writes MB Control Register to set doorbell 31521e9f767SBen Widawsky * 5. Caller either polls for doorbell to be clear or waits for interrupt if configured 31621e9f767SBen Widawsky * 6. Caller reads MB Status Register to fetch Return code 31721e9f767SBen Widawsky * 7. If command successful, Caller reads Command Register to get Payload Length 31821e9f767SBen Widawsky * 8. If output payload is non-empty, host reads Command Payload Registers 31921e9f767SBen Widawsky * 32021e9f767SBen Widawsky * Hardware is free to do whatever it wants before the doorbell is rung, 32121e9f767SBen Widawsky * and isn't allowed to change anything after it clears the doorbell. As 32221e9f767SBen Widawsky * such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can 32321e9f767SBen Widawsky * also happen in any order (though some orders might not make sense). 32421e9f767SBen Widawsky */ 32521e9f767SBen Widawsky 32621e9f767SBen Widawsky /* #1 */ 32721e9f767SBen Widawsky if (cxl_doorbell_busy(cxlm)) { 32821e9f767SBen Widawsky dev_err_ratelimited(&cxlm->pdev->dev, 32921e9f767SBen Widawsky "Mailbox re-busy after acquiring\n"); 33021e9f767SBen Widawsky return -EBUSY; 33121e9f767SBen Widawsky } 33221e9f767SBen Widawsky 33321e9f767SBen Widawsky cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK, 33421e9f767SBen Widawsky mbox_cmd->opcode); 33521e9f767SBen Widawsky if (mbox_cmd->size_in) { 33621e9f767SBen Widawsky if (WARN_ON(!mbox_cmd->payload_in)) 33721e9f767SBen Widawsky return -EINVAL; 33821e9f767SBen Widawsky 33921e9f767SBen Widawsky cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, 34021e9f767SBen Widawsky mbox_cmd->size_in); 34121e9f767SBen Widawsky memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in); 34221e9f767SBen Widawsky } 34321e9f767SBen Widawsky 34421e9f767SBen Widawsky /* #2, #3 */ 34521e9f767SBen Widawsky writeq(cmd_reg, cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 34621e9f767SBen Widawsky 34721e9f767SBen Widawsky /* #4 */ 34821e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, "Sending command\n"); 34921e9f767SBen Widawsky writel(CXLDEV_MBOX_CTRL_DOORBELL, 35021e9f767SBen Widawsky cxlm->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET); 35121e9f767SBen Widawsky 35221e9f767SBen Widawsky /* #5 */ 35321e9f767SBen Widawsky rc = cxl_mem_wait_for_doorbell(cxlm); 35421e9f767SBen Widawsky if (rc == -ETIMEDOUT) { 35521e9f767SBen Widawsky cxl_mem_mbox_timeout(cxlm, mbox_cmd); 35621e9f767SBen Widawsky return rc; 35721e9f767SBen Widawsky } 35821e9f767SBen Widawsky 35921e9f767SBen Widawsky /* #6 */ 36021e9f767SBen Widawsky status_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET); 36121e9f767SBen Widawsky mbox_cmd->return_code = 36221e9f767SBen Widawsky FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg); 36321e9f767SBen Widawsky 36421e9f767SBen Widawsky if (mbox_cmd->return_code != 0) { 36521e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, "Mailbox operation had an error\n"); 36621e9f767SBen Widawsky return 0; 36721e9f767SBen Widawsky } 36821e9f767SBen Widawsky 36921e9f767SBen Widawsky /* #7 */ 37021e9f767SBen Widawsky cmd_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); 37121e9f767SBen Widawsky out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg); 37221e9f767SBen Widawsky 37321e9f767SBen Widawsky /* #8 */ 37421e9f767SBen Widawsky if (out_len && mbox_cmd->payload_out) { 37521e9f767SBen Widawsky /* 37621e9f767SBen Widawsky * Sanitize the copy. If hardware misbehaves, out_len per the 37721e9f767SBen Widawsky * spec can actually be greater than the max allowed size (21 37821e9f767SBen Widawsky * bits available but spec defined 1M max). The caller also may 37921e9f767SBen Widawsky * have requested less data than the hardware supplied even 38021e9f767SBen Widawsky * within spec. 38121e9f767SBen Widawsky */ 38221e9f767SBen Widawsky size_t n = min3(mbox_cmd->size_out, cxlm->payload_size, out_len); 38321e9f767SBen Widawsky 38421e9f767SBen Widawsky memcpy_fromio(mbox_cmd->payload_out, payload, n); 38521e9f767SBen Widawsky mbox_cmd->size_out = n; 38621e9f767SBen Widawsky } else { 38721e9f767SBen Widawsky mbox_cmd->size_out = 0; 38821e9f767SBen Widawsky } 38921e9f767SBen Widawsky 39021e9f767SBen Widawsky return 0; 39121e9f767SBen Widawsky } 39221e9f767SBen Widawsky 39321e9f767SBen Widawsky /** 39421e9f767SBen Widawsky * cxl_mem_mbox_get() - Acquire exclusive access to the mailbox. 39521e9f767SBen Widawsky * @cxlm: The memory device to gain access to. 39621e9f767SBen Widawsky * 39721e9f767SBen Widawsky * Context: Any context. Takes the mbox_mutex. 39821e9f767SBen Widawsky * Return: 0 if exclusive access was acquired. 39921e9f767SBen Widawsky */ 40021e9f767SBen Widawsky static int cxl_mem_mbox_get(struct cxl_mem *cxlm) 40121e9f767SBen Widawsky { 40221e9f767SBen Widawsky struct device *dev = &cxlm->pdev->dev; 40321e9f767SBen Widawsky u64 md_status; 40421e9f767SBen Widawsky int rc; 40521e9f767SBen Widawsky 40621e9f767SBen Widawsky mutex_lock_io(&cxlm->mbox_mutex); 40721e9f767SBen Widawsky 40821e9f767SBen Widawsky /* 40921e9f767SBen Widawsky * XXX: There is some amount of ambiguity in the 2.0 version of the spec 41021e9f767SBen Widawsky * around the mailbox interface ready (8.2.8.5.1.1). The purpose of the 41121e9f767SBen Widawsky * bit is to allow firmware running on the device to notify the driver 41221e9f767SBen Widawsky * that it's ready to receive commands. It is unclear if the bit needs 41321e9f767SBen Widawsky * to be read for each transaction mailbox, ie. the firmware can switch 41421e9f767SBen Widawsky * it on and off as needed. Second, there is no defined timeout for 41521e9f767SBen Widawsky * mailbox ready, like there is for the doorbell interface. 41621e9f767SBen Widawsky * 41721e9f767SBen Widawsky * Assumptions: 41821e9f767SBen Widawsky * 1. The firmware might toggle the Mailbox Interface Ready bit, check 41921e9f767SBen Widawsky * it for every command. 42021e9f767SBen Widawsky * 42121e9f767SBen Widawsky * 2. If the doorbell is clear, the firmware should have first set the 42221e9f767SBen Widawsky * Mailbox Interface Ready bit. Therefore, waiting for the doorbell 42321e9f767SBen Widawsky * to be ready is sufficient. 42421e9f767SBen Widawsky */ 42521e9f767SBen Widawsky rc = cxl_mem_wait_for_doorbell(cxlm); 42621e9f767SBen Widawsky if (rc) { 42721e9f767SBen Widawsky dev_warn(dev, "Mailbox interface not ready\n"); 42821e9f767SBen Widawsky goto out; 42921e9f767SBen Widawsky } 43021e9f767SBen Widawsky 43121e9f767SBen Widawsky md_status = readq(cxlm->regs.memdev + CXLMDEV_STATUS_OFFSET); 43221e9f767SBen Widawsky if (!(md_status & CXLMDEV_MBOX_IF_READY && CXLMDEV_READY(md_status))) { 43321e9f767SBen Widawsky dev_err(dev, "mbox: reported doorbell ready, but not mbox ready\n"); 43421e9f767SBen Widawsky rc = -EBUSY; 43521e9f767SBen Widawsky goto out; 43621e9f767SBen Widawsky } 43721e9f767SBen Widawsky 43821e9f767SBen Widawsky /* 43921e9f767SBen Widawsky * Hardware shouldn't allow a ready status but also have failure bits 44021e9f767SBen Widawsky * set. Spit out an error, this should be a bug report 44121e9f767SBen Widawsky */ 44221e9f767SBen Widawsky rc = -EFAULT; 44321e9f767SBen Widawsky if (md_status & CXLMDEV_DEV_FATAL) { 44421e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but fatal\n"); 44521e9f767SBen Widawsky goto out; 44621e9f767SBen Widawsky } 44721e9f767SBen Widawsky if (md_status & CXLMDEV_FW_HALT) { 44821e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but halted\n"); 44921e9f767SBen Widawsky goto out; 45021e9f767SBen Widawsky } 45121e9f767SBen Widawsky if (CXLMDEV_RESET_NEEDED(md_status)) { 45221e9f767SBen Widawsky dev_err(dev, "mbox: reported ready, but reset needed\n"); 45321e9f767SBen Widawsky goto out; 45421e9f767SBen Widawsky } 45521e9f767SBen Widawsky 45621e9f767SBen Widawsky /* with lock held */ 45721e9f767SBen Widawsky return 0; 45821e9f767SBen Widawsky 45921e9f767SBen Widawsky out: 46021e9f767SBen Widawsky mutex_unlock(&cxlm->mbox_mutex); 46121e9f767SBen Widawsky return rc; 46221e9f767SBen Widawsky } 46321e9f767SBen Widawsky 46421e9f767SBen Widawsky /** 46521e9f767SBen Widawsky * cxl_mem_mbox_put() - Release exclusive access to the mailbox. 46621e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 46721e9f767SBen Widawsky * 46821e9f767SBen Widawsky * Context: Any context. Expects mbox_mutex to be held. 46921e9f767SBen Widawsky */ 47021e9f767SBen Widawsky static void cxl_mem_mbox_put(struct cxl_mem *cxlm) 47121e9f767SBen Widawsky { 47221e9f767SBen Widawsky mutex_unlock(&cxlm->mbox_mutex); 47321e9f767SBen Widawsky } 47421e9f767SBen Widawsky 47521e9f767SBen Widawsky /** 47621e9f767SBen Widawsky * handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace. 47721e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 47821e9f767SBen Widawsky * @cmd: The validated command. 47921e9f767SBen Widawsky * @in_payload: Pointer to userspace's input payload. 48021e9f767SBen Widawsky * @out_payload: Pointer to userspace's output payload. 48121e9f767SBen Widawsky * @size_out: (Input) Max payload size to copy out. 48221e9f767SBen Widawsky * (Output) Payload size hardware generated. 48321e9f767SBen Widawsky * @retval: Hardware generated return code from the operation. 48421e9f767SBen Widawsky * 48521e9f767SBen Widawsky * Return: 48621e9f767SBen Widawsky * * %0 - Mailbox transaction succeeded. This implies the mailbox 48721e9f767SBen Widawsky * protocol completed successfully not that the operation itself 48821e9f767SBen Widawsky * was successful. 48921e9f767SBen Widawsky * * %-ENOMEM - Couldn't allocate a bounce buffer. 49021e9f767SBen Widawsky * * %-EFAULT - Something happened with copy_to/from_user. 49121e9f767SBen Widawsky * * %-EINTR - Mailbox acquisition interrupted. 49221e9f767SBen Widawsky * * %-EXXX - Transaction level failures. 49321e9f767SBen Widawsky * 49421e9f767SBen Widawsky * Creates the appropriate mailbox command and dispatches it on behalf of a 49521e9f767SBen Widawsky * userspace request. The input and output payloads are copied between 49621e9f767SBen Widawsky * userspace. 49721e9f767SBen Widawsky * 49821e9f767SBen Widawsky * See cxl_send_cmd(). 49921e9f767SBen Widawsky */ 50021e9f767SBen Widawsky static int handle_mailbox_cmd_from_user(struct cxl_mem *cxlm, 50121e9f767SBen Widawsky const struct cxl_mem_command *cmd, 50221e9f767SBen Widawsky u64 in_payload, u64 out_payload, 50321e9f767SBen Widawsky s32 *size_out, u32 *retval) 50421e9f767SBen Widawsky { 50521e9f767SBen Widawsky struct device *dev = &cxlm->pdev->dev; 50621e9f767SBen Widawsky struct mbox_cmd mbox_cmd = { 50721e9f767SBen Widawsky .opcode = cmd->opcode, 50821e9f767SBen Widawsky .size_in = cmd->info.size_in, 50921e9f767SBen Widawsky .size_out = cmd->info.size_out, 51021e9f767SBen Widawsky }; 51121e9f767SBen Widawsky int rc; 51221e9f767SBen Widawsky 51321e9f767SBen Widawsky if (cmd->info.size_out) { 51421e9f767SBen Widawsky mbox_cmd.payload_out = kvzalloc(cmd->info.size_out, GFP_KERNEL); 51521e9f767SBen Widawsky if (!mbox_cmd.payload_out) 51621e9f767SBen Widawsky return -ENOMEM; 51721e9f767SBen Widawsky } 51821e9f767SBen Widawsky 51921e9f767SBen Widawsky if (cmd->info.size_in) { 52021e9f767SBen Widawsky mbox_cmd.payload_in = vmemdup_user(u64_to_user_ptr(in_payload), 52121e9f767SBen Widawsky cmd->info.size_in); 52221e9f767SBen Widawsky if (IS_ERR(mbox_cmd.payload_in)) { 52321e9f767SBen Widawsky kvfree(mbox_cmd.payload_out); 52421e9f767SBen Widawsky return PTR_ERR(mbox_cmd.payload_in); 52521e9f767SBen Widawsky } 52621e9f767SBen Widawsky } 52721e9f767SBen Widawsky 52821e9f767SBen Widawsky rc = cxl_mem_mbox_get(cxlm); 52921e9f767SBen Widawsky if (rc) 53021e9f767SBen Widawsky goto out; 53121e9f767SBen Widawsky 53221e9f767SBen Widawsky dev_dbg(dev, 53321e9f767SBen Widawsky "Submitting %s command for user\n" 53421e9f767SBen Widawsky "\topcode: %x\n" 53521e9f767SBen Widawsky "\tsize: %ub\n", 53621e9f767SBen Widawsky cxl_command_names[cmd->info.id].name, mbox_cmd.opcode, 53721e9f767SBen Widawsky cmd->info.size_in); 53821e9f767SBen Widawsky 53921e9f767SBen Widawsky dev_WARN_ONCE(dev, cmd->info.id == CXL_MEM_COMMAND_ID_RAW, 54021e9f767SBen Widawsky "raw command path used\n"); 54121e9f767SBen Widawsky 54221e9f767SBen Widawsky rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd); 54321e9f767SBen Widawsky cxl_mem_mbox_put(cxlm); 54421e9f767SBen Widawsky if (rc) 54521e9f767SBen Widawsky goto out; 54621e9f767SBen Widawsky 54721e9f767SBen Widawsky /* 54821e9f767SBen Widawsky * @size_out contains the max size that's allowed to be written back out 54921e9f767SBen Widawsky * to userspace. While the payload may have written more output than 55021e9f767SBen Widawsky * this it will have to be ignored. 55121e9f767SBen Widawsky */ 55221e9f767SBen Widawsky if (mbox_cmd.size_out) { 55321e9f767SBen Widawsky dev_WARN_ONCE(dev, mbox_cmd.size_out > *size_out, 55421e9f767SBen Widawsky "Invalid return size\n"); 55521e9f767SBen Widawsky if (copy_to_user(u64_to_user_ptr(out_payload), 55621e9f767SBen Widawsky mbox_cmd.payload_out, mbox_cmd.size_out)) { 55721e9f767SBen Widawsky rc = -EFAULT; 55821e9f767SBen Widawsky goto out; 55921e9f767SBen Widawsky } 56021e9f767SBen Widawsky } 56121e9f767SBen Widawsky 56221e9f767SBen Widawsky *size_out = mbox_cmd.size_out; 56321e9f767SBen Widawsky *retval = mbox_cmd.return_code; 56421e9f767SBen Widawsky 56521e9f767SBen Widawsky out: 56621e9f767SBen Widawsky kvfree(mbox_cmd.payload_in); 56721e9f767SBen Widawsky kvfree(mbox_cmd.payload_out); 56821e9f767SBen Widawsky return rc; 56921e9f767SBen Widawsky } 57021e9f767SBen Widawsky 57121e9f767SBen Widawsky static bool cxl_mem_raw_command_allowed(u16 opcode) 57221e9f767SBen Widawsky { 57321e9f767SBen Widawsky int i; 57421e9f767SBen Widawsky 57521e9f767SBen Widawsky if (!IS_ENABLED(CONFIG_CXL_MEM_RAW_COMMANDS)) 57621e9f767SBen Widawsky return false; 57721e9f767SBen Widawsky 578*9e56614cSDan Williams if (security_locked_down(LOCKDOWN_PCI_ACCESS)) 57921e9f767SBen Widawsky return false; 58021e9f767SBen Widawsky 58121e9f767SBen Widawsky if (cxl_raw_allow_all) 58221e9f767SBen Widawsky return true; 58321e9f767SBen Widawsky 58421e9f767SBen Widawsky if (cxl_is_security_command(opcode)) 58521e9f767SBen Widawsky return false; 58621e9f767SBen Widawsky 58721e9f767SBen Widawsky for (i = 0; i < ARRAY_SIZE(cxl_disabled_raw_commands); i++) 58821e9f767SBen Widawsky if (cxl_disabled_raw_commands[i] == opcode) 58921e9f767SBen Widawsky return false; 59021e9f767SBen Widawsky 59121e9f767SBen Widawsky return true; 59221e9f767SBen Widawsky } 59321e9f767SBen Widawsky 59421e9f767SBen Widawsky /** 59521e9f767SBen Widawsky * cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND. 59621e9f767SBen Widawsky * @cxlm: &struct cxl_mem device whose mailbox will be used. 59721e9f767SBen Widawsky * @send_cmd: &struct cxl_send_command copied in from userspace. 59821e9f767SBen Widawsky * @out_cmd: Sanitized and populated &struct cxl_mem_command. 59921e9f767SBen Widawsky * 60021e9f767SBen Widawsky * Return: 60121e9f767SBen Widawsky * * %0 - @out_cmd is ready to send. 60221e9f767SBen Widawsky * * %-ENOTTY - Invalid command specified. 60321e9f767SBen Widawsky * * %-EINVAL - Reserved fields or invalid values were used. 60421e9f767SBen Widawsky * * %-ENOMEM - Input or output buffer wasn't sized properly. 60521e9f767SBen Widawsky * * %-EPERM - Attempted to use a protected command. 60621e9f767SBen Widawsky * 60721e9f767SBen Widawsky * The result of this command is a fully validated command in @out_cmd that is 60821e9f767SBen Widawsky * safe to send to the hardware. 60921e9f767SBen Widawsky * 61021e9f767SBen Widawsky * See handle_mailbox_cmd_from_user() 61121e9f767SBen Widawsky */ 61221e9f767SBen Widawsky static int cxl_validate_cmd_from_user(struct cxl_mem *cxlm, 61321e9f767SBen Widawsky const struct cxl_send_command *send_cmd, 61421e9f767SBen Widawsky struct cxl_mem_command *out_cmd) 61521e9f767SBen Widawsky { 61621e9f767SBen Widawsky const struct cxl_command_info *info; 61721e9f767SBen Widawsky struct cxl_mem_command *c; 61821e9f767SBen Widawsky 61921e9f767SBen Widawsky if (send_cmd->id == 0 || send_cmd->id >= CXL_MEM_COMMAND_ID_MAX) 62021e9f767SBen Widawsky return -ENOTTY; 62121e9f767SBen Widawsky 62221e9f767SBen Widawsky /* 62321e9f767SBen Widawsky * The user can never specify an input payload larger than what hardware 62421e9f767SBen Widawsky * supports, but output can be arbitrarily large (simply write out as 62521e9f767SBen Widawsky * much data as the hardware provides). 62621e9f767SBen Widawsky */ 62721e9f767SBen Widawsky if (send_cmd->in.size > cxlm->payload_size) 62821e9f767SBen Widawsky return -EINVAL; 62921e9f767SBen Widawsky 63021e9f767SBen Widawsky /* 63121e9f767SBen Widawsky * Checks are bypassed for raw commands but a WARN/taint will occur 63221e9f767SBen Widawsky * later in the callchain 63321e9f767SBen Widawsky */ 63421e9f767SBen Widawsky if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW) { 63521e9f767SBen Widawsky const struct cxl_mem_command temp = { 63621e9f767SBen Widawsky .info = { 63721e9f767SBen Widawsky .id = CXL_MEM_COMMAND_ID_RAW, 63821e9f767SBen Widawsky .flags = 0, 63921e9f767SBen Widawsky .size_in = send_cmd->in.size, 64021e9f767SBen Widawsky .size_out = send_cmd->out.size, 64121e9f767SBen Widawsky }, 64221e9f767SBen Widawsky .opcode = send_cmd->raw.opcode 64321e9f767SBen Widawsky }; 64421e9f767SBen Widawsky 64521e9f767SBen Widawsky if (send_cmd->raw.rsvd) 64621e9f767SBen Widawsky return -EINVAL; 64721e9f767SBen Widawsky 64821e9f767SBen Widawsky /* 64921e9f767SBen Widawsky * Unlike supported commands, the output size of RAW commands 65021e9f767SBen Widawsky * gets passed along without further checking, so it must be 65121e9f767SBen Widawsky * validated here. 65221e9f767SBen Widawsky */ 65321e9f767SBen Widawsky if (send_cmd->out.size > cxlm->payload_size) 65421e9f767SBen Widawsky return -EINVAL; 65521e9f767SBen Widawsky 65621e9f767SBen Widawsky if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode)) 65721e9f767SBen Widawsky return -EPERM; 65821e9f767SBen Widawsky 65921e9f767SBen Widawsky memcpy(out_cmd, &temp, sizeof(temp)); 66021e9f767SBen Widawsky 66121e9f767SBen Widawsky return 0; 66221e9f767SBen Widawsky } 66321e9f767SBen Widawsky 66421e9f767SBen Widawsky if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK) 66521e9f767SBen Widawsky return -EINVAL; 66621e9f767SBen Widawsky 66721e9f767SBen Widawsky if (send_cmd->rsvd) 66821e9f767SBen Widawsky return -EINVAL; 66921e9f767SBen Widawsky 67021e9f767SBen Widawsky if (send_cmd->in.rsvd || send_cmd->out.rsvd) 67121e9f767SBen Widawsky return -EINVAL; 67221e9f767SBen Widawsky 67321e9f767SBen Widawsky /* Convert user's command into the internal representation */ 67421e9f767SBen Widawsky c = &mem_commands[send_cmd->id]; 67521e9f767SBen Widawsky info = &c->info; 67621e9f767SBen Widawsky 67721e9f767SBen Widawsky /* Check that the command is enabled for hardware */ 67821e9f767SBen Widawsky if (!test_bit(info->id, cxlm->enabled_cmds)) 67921e9f767SBen Widawsky return -ENOTTY; 68021e9f767SBen Widawsky 68121e9f767SBen Widawsky /* Check the input buffer is the expected size */ 68221e9f767SBen Widawsky if (info->size_in >= 0 && info->size_in != send_cmd->in.size) 68321e9f767SBen Widawsky return -ENOMEM; 68421e9f767SBen Widawsky 68521e9f767SBen Widawsky /* Check the output buffer is at least large enough */ 68621e9f767SBen Widawsky if (info->size_out >= 0 && send_cmd->out.size < info->size_out) 68721e9f767SBen Widawsky return -ENOMEM; 68821e9f767SBen Widawsky 68921e9f767SBen Widawsky memcpy(out_cmd, c, sizeof(*c)); 69021e9f767SBen Widawsky out_cmd->info.size_in = send_cmd->in.size; 69121e9f767SBen Widawsky /* 69221e9f767SBen Widawsky * XXX: out_cmd->info.size_out will be controlled by the driver, and the 69321e9f767SBen Widawsky * specified number of bytes @send_cmd->out.size will be copied back out 69421e9f767SBen Widawsky * to userspace. 69521e9f767SBen Widawsky */ 69621e9f767SBen Widawsky 69721e9f767SBen Widawsky return 0; 69821e9f767SBen Widawsky } 69921e9f767SBen Widawsky 70021e9f767SBen Widawsky static int cxl_query_cmd(struct cxl_memdev *cxlmd, 70121e9f767SBen Widawsky struct cxl_mem_query_commands __user *q) 70221e9f767SBen Widawsky { 70321e9f767SBen Widawsky struct device *dev = &cxlmd->dev; 70421e9f767SBen Widawsky struct cxl_mem_command *cmd; 70521e9f767SBen Widawsky u32 n_commands; 70621e9f767SBen Widawsky int j = 0; 70721e9f767SBen Widawsky 70821e9f767SBen Widawsky dev_dbg(dev, "Query IOCTL\n"); 70921e9f767SBen Widawsky 71021e9f767SBen Widawsky if (get_user(n_commands, &q->n_commands)) 71121e9f767SBen Widawsky return -EFAULT; 71221e9f767SBen Widawsky 71321e9f767SBen Widawsky /* returns the total number if 0 elements are requested. */ 71421e9f767SBen Widawsky if (n_commands == 0) 71521e9f767SBen Widawsky return put_user(cxl_cmd_count, &q->n_commands); 71621e9f767SBen Widawsky 71721e9f767SBen Widawsky /* 71821e9f767SBen Widawsky * otherwise, return max(n_commands, total commands) cxl_command_info 71921e9f767SBen Widawsky * structures. 72021e9f767SBen Widawsky */ 72121e9f767SBen Widawsky cxl_for_each_cmd(cmd) { 72221e9f767SBen Widawsky const struct cxl_command_info *info = &cmd->info; 72321e9f767SBen Widawsky 72421e9f767SBen Widawsky if (copy_to_user(&q->commands[j++], info, sizeof(*info))) 72521e9f767SBen Widawsky return -EFAULT; 72621e9f767SBen Widawsky 72721e9f767SBen Widawsky if (j == n_commands) 72821e9f767SBen Widawsky break; 72921e9f767SBen Widawsky } 73021e9f767SBen Widawsky 73121e9f767SBen Widawsky return 0; 73221e9f767SBen Widawsky } 73321e9f767SBen Widawsky 73421e9f767SBen Widawsky static int cxl_send_cmd(struct cxl_memdev *cxlmd, 73521e9f767SBen Widawsky struct cxl_send_command __user *s) 73621e9f767SBen Widawsky { 73721e9f767SBen Widawsky struct cxl_mem *cxlm = cxlmd->cxlm; 73821e9f767SBen Widawsky struct device *dev = &cxlmd->dev; 73921e9f767SBen Widawsky struct cxl_send_command send; 74021e9f767SBen Widawsky struct cxl_mem_command c; 74121e9f767SBen Widawsky int rc; 74221e9f767SBen Widawsky 74321e9f767SBen Widawsky dev_dbg(dev, "Send IOCTL\n"); 74421e9f767SBen Widawsky 74521e9f767SBen Widawsky if (copy_from_user(&send, s, sizeof(send))) 74621e9f767SBen Widawsky return -EFAULT; 74721e9f767SBen Widawsky 74821e9f767SBen Widawsky rc = cxl_validate_cmd_from_user(cxlmd->cxlm, &send, &c); 74921e9f767SBen Widawsky if (rc) 75021e9f767SBen Widawsky return rc; 75121e9f767SBen Widawsky 75221e9f767SBen Widawsky /* Prepare to handle a full payload for variable sized output */ 75321e9f767SBen Widawsky if (c.info.size_out < 0) 75421e9f767SBen Widawsky c.info.size_out = cxlm->payload_size; 75521e9f767SBen Widawsky 75621e9f767SBen Widawsky rc = handle_mailbox_cmd_from_user(cxlm, &c, send.in.payload, 75721e9f767SBen Widawsky send.out.payload, &send.out.size, 75821e9f767SBen Widawsky &send.retval); 75921e9f767SBen Widawsky if (rc) 76021e9f767SBen Widawsky return rc; 76121e9f767SBen Widawsky 76221e9f767SBen Widawsky if (copy_to_user(s, &send, sizeof(send))) 76321e9f767SBen Widawsky return -EFAULT; 76421e9f767SBen Widawsky 76521e9f767SBen Widawsky return 0; 76621e9f767SBen Widawsky } 76721e9f767SBen Widawsky 76821e9f767SBen Widawsky static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, 76921e9f767SBen Widawsky unsigned long arg) 77021e9f767SBen Widawsky { 77121e9f767SBen Widawsky switch (cmd) { 77221e9f767SBen Widawsky case CXL_MEM_QUERY_COMMANDS: 77321e9f767SBen Widawsky return cxl_query_cmd(cxlmd, (void __user *)arg); 77421e9f767SBen Widawsky case CXL_MEM_SEND_COMMAND: 77521e9f767SBen Widawsky return cxl_send_cmd(cxlmd, (void __user *)arg); 77621e9f767SBen Widawsky default: 77721e9f767SBen Widawsky return -ENOTTY; 77821e9f767SBen Widawsky } 77921e9f767SBen Widawsky } 78021e9f767SBen Widawsky 78121e9f767SBen Widawsky static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, 78221e9f767SBen Widawsky unsigned long arg) 78321e9f767SBen Widawsky { 78421e9f767SBen Widawsky struct cxl_memdev *cxlmd = file->private_data; 78521e9f767SBen Widawsky int rc = -ENXIO; 78621e9f767SBen Widawsky 78721e9f767SBen Widawsky down_read(&cxl_memdev_rwsem); 78821e9f767SBen Widawsky if (cxlmd->cxlm) 78921e9f767SBen Widawsky rc = __cxl_memdev_ioctl(cxlmd, cmd, arg); 79021e9f767SBen Widawsky up_read(&cxl_memdev_rwsem); 79121e9f767SBen Widawsky 79221e9f767SBen Widawsky return rc; 79321e9f767SBen Widawsky } 79421e9f767SBen Widawsky 79521e9f767SBen Widawsky static int cxl_memdev_open(struct inode *inode, struct file *file) 79621e9f767SBen Widawsky { 79721e9f767SBen Widawsky struct cxl_memdev *cxlmd = 79821e9f767SBen Widawsky container_of(inode->i_cdev, typeof(*cxlmd), cdev); 79921e9f767SBen Widawsky 80021e9f767SBen Widawsky get_device(&cxlmd->dev); 80121e9f767SBen Widawsky file->private_data = cxlmd; 80221e9f767SBen Widawsky 80321e9f767SBen Widawsky return 0; 80421e9f767SBen Widawsky } 80521e9f767SBen Widawsky 80621e9f767SBen Widawsky static int cxl_memdev_release_file(struct inode *inode, struct file *file) 80721e9f767SBen Widawsky { 80821e9f767SBen Widawsky struct cxl_memdev *cxlmd = 80921e9f767SBen Widawsky container_of(inode->i_cdev, typeof(*cxlmd), cdev); 81021e9f767SBen Widawsky 81121e9f767SBen Widawsky put_device(&cxlmd->dev); 81221e9f767SBen Widawsky 81321e9f767SBen Widawsky return 0; 81421e9f767SBen Widawsky } 81521e9f767SBen Widawsky 8169cc238c7SDan Williams static void cxl_memdev_shutdown(struct device *dev) 8179cc238c7SDan Williams { 8189cc238c7SDan Williams struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 8199cc238c7SDan Williams 8209cc238c7SDan Williams down_write(&cxl_memdev_rwsem); 8219cc238c7SDan Williams cxlmd->cxlm = NULL; 8229cc238c7SDan Williams up_write(&cxl_memdev_rwsem); 8239cc238c7SDan Williams } 8249cc238c7SDan Williams 8259cc238c7SDan Williams static const struct cdevm_file_operations cxl_memdev_fops = { 8269cc238c7SDan Williams .fops = { 82721e9f767SBen Widawsky .owner = THIS_MODULE, 82821e9f767SBen Widawsky .unlocked_ioctl = cxl_memdev_ioctl, 82921e9f767SBen Widawsky .open = cxl_memdev_open, 83021e9f767SBen Widawsky .release = cxl_memdev_release_file, 83121e9f767SBen Widawsky .compat_ioctl = compat_ptr_ioctl, 83221e9f767SBen Widawsky .llseek = noop_llseek, 8339cc238c7SDan Williams }, 8349cc238c7SDan Williams .shutdown = cxl_memdev_shutdown, 83521e9f767SBen Widawsky }; 83621e9f767SBen Widawsky 83721e9f767SBen Widawsky static inline struct cxl_mem_command *cxl_mem_find_command(u16 opcode) 83821e9f767SBen Widawsky { 83921e9f767SBen Widawsky struct cxl_mem_command *c; 84021e9f767SBen Widawsky 84121e9f767SBen Widawsky cxl_for_each_cmd(c) 84221e9f767SBen Widawsky if (c->opcode == opcode) 84321e9f767SBen Widawsky return c; 84421e9f767SBen Widawsky 84521e9f767SBen Widawsky return NULL; 84621e9f767SBen Widawsky } 84721e9f767SBen Widawsky 84821e9f767SBen Widawsky /** 84921e9f767SBen Widawsky * cxl_mem_mbox_send_cmd() - Send a mailbox command to a memory device. 85021e9f767SBen Widawsky * @cxlm: The CXL memory device to communicate with. 85121e9f767SBen Widawsky * @opcode: Opcode for the mailbox command. 85221e9f767SBen Widawsky * @in: The input payload for the mailbox command. 85321e9f767SBen Widawsky * @in_size: The length of the input payload 85421e9f767SBen Widawsky * @out: Caller allocated buffer for the output. 85521e9f767SBen Widawsky * @out_size: Expected size of output. 85621e9f767SBen Widawsky * 85721e9f767SBen Widawsky * Context: Any context. Will acquire and release mbox_mutex. 85821e9f767SBen Widawsky * Return: 85921e9f767SBen Widawsky * * %>=0 - Number of bytes returned in @out. 86021e9f767SBen Widawsky * * %-E2BIG - Payload is too large for hardware. 86121e9f767SBen Widawsky * * %-EBUSY - Couldn't acquire exclusive mailbox access. 86221e9f767SBen Widawsky * * %-EFAULT - Hardware error occurred. 86321e9f767SBen Widawsky * * %-ENXIO - Command completed, but device reported an error. 86421e9f767SBen Widawsky * * %-EIO - Unexpected output size. 86521e9f767SBen Widawsky * 86621e9f767SBen Widawsky * Mailbox commands may execute successfully yet the device itself reported an 86721e9f767SBen Widawsky * error. While this distinction can be useful for commands from userspace, the 86821e9f767SBen Widawsky * kernel will only be able to use results when both are successful. 86921e9f767SBen Widawsky * 87021e9f767SBen Widawsky * See __cxl_mem_mbox_send_cmd() 87121e9f767SBen Widawsky */ 87221e9f767SBen Widawsky static int cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, u16 opcode, 87321e9f767SBen Widawsky void *in, size_t in_size, 87421e9f767SBen Widawsky void *out, size_t out_size) 87521e9f767SBen Widawsky { 87621e9f767SBen Widawsky const struct cxl_mem_command *cmd = cxl_mem_find_command(opcode); 87721e9f767SBen Widawsky struct mbox_cmd mbox_cmd = { 87821e9f767SBen Widawsky .opcode = opcode, 87921e9f767SBen Widawsky .payload_in = in, 88021e9f767SBen Widawsky .size_in = in_size, 88121e9f767SBen Widawsky .size_out = out_size, 88221e9f767SBen Widawsky .payload_out = out, 88321e9f767SBen Widawsky }; 88421e9f767SBen Widawsky int rc; 88521e9f767SBen Widawsky 88621e9f767SBen Widawsky if (out_size > cxlm->payload_size) 88721e9f767SBen Widawsky return -E2BIG; 88821e9f767SBen Widawsky 88921e9f767SBen Widawsky rc = cxl_mem_mbox_get(cxlm); 89021e9f767SBen Widawsky if (rc) 89121e9f767SBen Widawsky return rc; 89221e9f767SBen Widawsky 89321e9f767SBen Widawsky rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd); 89421e9f767SBen Widawsky cxl_mem_mbox_put(cxlm); 89521e9f767SBen Widawsky if (rc) 89621e9f767SBen Widawsky return rc; 89721e9f767SBen Widawsky 89821e9f767SBen Widawsky /* TODO: Map return code to proper kernel style errno */ 89921e9f767SBen Widawsky if (mbox_cmd.return_code != CXL_MBOX_SUCCESS) 90021e9f767SBen Widawsky return -ENXIO; 90121e9f767SBen Widawsky 90221e9f767SBen Widawsky /* 90321e9f767SBen Widawsky * Variable sized commands can't be validated and so it's up to the 90421e9f767SBen Widawsky * caller to do that if they wish. 90521e9f767SBen Widawsky */ 90621e9f767SBen Widawsky if (cmd->info.size_out >= 0 && mbox_cmd.size_out != out_size) 90721e9f767SBen Widawsky return -EIO; 90821e9f767SBen Widawsky 90921e9f767SBen Widawsky return 0; 91021e9f767SBen Widawsky } 91121e9f767SBen Widawsky 91221e9f767SBen Widawsky static int cxl_mem_setup_mailbox(struct cxl_mem *cxlm) 91321e9f767SBen Widawsky { 91421e9f767SBen Widawsky const int cap = readl(cxlm->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET); 91521e9f767SBen Widawsky 91621e9f767SBen Widawsky cxlm->payload_size = 91721e9f767SBen Widawsky 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap); 91821e9f767SBen Widawsky 91921e9f767SBen Widawsky /* 92021e9f767SBen Widawsky * CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register 92121e9f767SBen Widawsky * 92221e9f767SBen Widawsky * If the size is too small, mandatory commands will not work and so 92321e9f767SBen Widawsky * there's no point in going forward. If the size is too large, there's 92421e9f767SBen Widawsky * no harm is soft limiting it. 92521e9f767SBen Widawsky */ 92621e9f767SBen Widawsky cxlm->payload_size = min_t(size_t, cxlm->payload_size, SZ_1M); 92721e9f767SBen Widawsky if (cxlm->payload_size < 256) { 92821e9f767SBen Widawsky dev_err(&cxlm->pdev->dev, "Mailbox is too small (%zub)", 92921e9f767SBen Widawsky cxlm->payload_size); 93021e9f767SBen Widawsky return -ENXIO; 93121e9f767SBen Widawsky } 93221e9f767SBen Widawsky 93321e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, "Mailbox payload sized %zu", 93421e9f767SBen Widawsky cxlm->payload_size); 93521e9f767SBen Widawsky 93621e9f767SBen Widawsky return 0; 93721e9f767SBen Widawsky } 93821e9f767SBen Widawsky 9391b0a1a2aSBen Widawsky static struct cxl_mem *cxl_mem_create(struct pci_dev *pdev) 94021e9f767SBen Widawsky { 94121e9f767SBen Widawsky struct device *dev = &pdev->dev; 94221e9f767SBen Widawsky struct cxl_mem *cxlm; 94321e9f767SBen Widawsky 9445d0c6f02SBen Widawsky cxlm = devm_kzalloc(dev, sizeof(*cxlm), GFP_KERNEL); 94521e9f767SBen Widawsky if (!cxlm) { 94621e9f767SBen Widawsky dev_err(dev, "No memory available\n"); 9471b0a1a2aSBen Widawsky return ERR_PTR(-ENOMEM); 94821e9f767SBen Widawsky } 94921e9f767SBen Widawsky 9501b0a1a2aSBen Widawsky mutex_init(&cxlm->mbox_mutex); 9511b0a1a2aSBen Widawsky cxlm->pdev = pdev; 9521b0a1a2aSBen Widawsky cxlm->enabled_cmds = 9531b0a1a2aSBen Widawsky devm_kmalloc_array(dev, BITS_TO_LONGS(cxl_cmd_count), 9541b0a1a2aSBen Widawsky sizeof(unsigned long), 9551b0a1a2aSBen Widawsky GFP_KERNEL | __GFP_ZERO); 9561b0a1a2aSBen Widawsky if (!cxlm->enabled_cmds) { 9571b0a1a2aSBen Widawsky dev_err(dev, "No memory available for bitmap\n"); 9581b0a1a2aSBen Widawsky return ERR_PTR(-ENOMEM); 9591b0a1a2aSBen Widawsky } 9601b0a1a2aSBen Widawsky 9611b0a1a2aSBen Widawsky return cxlm; 9621b0a1a2aSBen Widawsky } 9631b0a1a2aSBen Widawsky 96407d62eacSIra Weiny static void __iomem *cxl_mem_map_regblock(struct cxl_mem *cxlm, 96507d62eacSIra Weiny u8 bar, u64 offset) 9661b0a1a2aSBen Widawsky { 9671b0a1a2aSBen Widawsky struct pci_dev *pdev = cxlm->pdev; 9681b0a1a2aSBen Widawsky struct device *dev = &pdev->dev; 969f8a7e8c2SIra Weiny void __iomem *addr; 9701b0a1a2aSBen Widawsky 97121e9f767SBen Widawsky /* Basic sanity check that BAR is big enough */ 97221e9f767SBen Widawsky if (pci_resource_len(pdev, bar) < offset) { 97321e9f767SBen Widawsky dev_err(dev, "BAR%d: %pr: too small (offset: %#llx)\n", bar, 97421e9f767SBen Widawsky &pdev->resource[bar], (unsigned long long)offset); 9756630d31cSBen Widawsky return IOMEM_ERR_PTR(-ENXIO); 97621e9f767SBen Widawsky } 97721e9f767SBen Widawsky 97830af9729SIra Weiny addr = pci_iomap(pdev, bar, 0); 979f8a7e8c2SIra Weiny if (!addr) { 98021e9f767SBen Widawsky dev_err(dev, "failed to map registers\n"); 981f8a7e8c2SIra Weiny return addr; 98221e9f767SBen Widawsky } 98321e9f767SBen Widawsky 984f8a7e8c2SIra Weiny dev_dbg(dev, "Mapped CXL Memory Device resource bar %u @ %#llx\n", 985f8a7e8c2SIra Weiny bar, offset); 9866630d31cSBen Widawsky 98730af9729SIra Weiny return addr; 98830af9729SIra Weiny } 98930af9729SIra Weiny 99030af9729SIra Weiny static void cxl_mem_unmap_regblock(struct cxl_mem *cxlm, void __iomem *base) 99130af9729SIra Weiny { 99230af9729SIra Weiny pci_iounmap(cxlm->pdev, base); 99321e9f767SBen Widawsky } 99421e9f767SBen Widawsky 99521e9f767SBen Widawsky static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec) 99621e9f767SBen Widawsky { 99721e9f767SBen Widawsky int pos; 99821e9f767SBen Widawsky 99921e9f767SBen Widawsky pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DVSEC); 100021e9f767SBen Widawsky if (!pos) 100121e9f767SBen Widawsky return 0; 100221e9f767SBen Widawsky 100321e9f767SBen Widawsky while (pos) { 100421e9f767SBen Widawsky u16 vendor, id; 100521e9f767SBen Widawsky 100621e9f767SBen Widawsky pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER1, &vendor); 100721e9f767SBen Widawsky pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER2, &id); 100821e9f767SBen Widawsky if (vendor == PCI_DVSEC_VENDOR_ID_CXL && dvsec == id) 100921e9f767SBen Widawsky return pos; 101021e9f767SBen Widawsky 101121e9f767SBen Widawsky pos = pci_find_next_ext_capability(pdev, pos, 101221e9f767SBen Widawsky PCI_EXT_CAP_ID_DVSEC); 101321e9f767SBen Widawsky } 101421e9f767SBen Widawsky 101521e9f767SBen Widawsky return 0; 101621e9f767SBen Widawsky } 101721e9f767SBen Widawsky 101830af9729SIra Weiny static int cxl_probe_regs(struct cxl_mem *cxlm, void __iomem *base, 101930af9729SIra Weiny struct cxl_register_map *map) 102030af9729SIra Weiny { 102130af9729SIra Weiny struct pci_dev *pdev = cxlm->pdev; 102230af9729SIra Weiny struct device *dev = &pdev->dev; 102308422378SBen Widawsky struct cxl_component_reg_map *comp_map; 102430af9729SIra Weiny struct cxl_device_reg_map *dev_map; 102530af9729SIra Weiny 102630af9729SIra Weiny switch (map->reg_type) { 102708422378SBen Widawsky case CXL_REGLOC_RBI_COMPONENT: 102808422378SBen Widawsky comp_map = &map->component_map; 102908422378SBen Widawsky cxl_probe_component_regs(dev, base, comp_map); 103008422378SBen Widawsky if (!comp_map->hdm_decoder.valid) { 103108422378SBen Widawsky dev_err(dev, "HDM decoder registers not found\n"); 103208422378SBen Widawsky return -ENXIO; 103308422378SBen Widawsky } 103408422378SBen Widawsky 103508422378SBen Widawsky dev_dbg(dev, "Set up component registers\n"); 103608422378SBen Widawsky break; 103730af9729SIra Weiny case CXL_REGLOC_RBI_MEMDEV: 103830af9729SIra Weiny dev_map = &map->device_map; 103930af9729SIra Weiny cxl_probe_device_regs(dev, base, dev_map); 104030af9729SIra Weiny if (!dev_map->status.valid || !dev_map->mbox.valid || 104130af9729SIra Weiny !dev_map->memdev.valid) { 104230af9729SIra Weiny dev_err(dev, "registers not found: %s%s%s\n", 104330af9729SIra Weiny !dev_map->status.valid ? "status " : "", 104430af9729SIra Weiny !dev_map->mbox.valid ? "status " : "", 104530af9729SIra Weiny !dev_map->memdev.valid ? "status " : ""); 104630af9729SIra Weiny return -ENXIO; 104730af9729SIra Weiny } 104830af9729SIra Weiny 104930af9729SIra Weiny dev_dbg(dev, "Probing device registers...\n"); 105030af9729SIra Weiny break; 105130af9729SIra Weiny default: 105230af9729SIra Weiny break; 105330af9729SIra Weiny } 105430af9729SIra Weiny 105530af9729SIra Weiny return 0; 105630af9729SIra Weiny } 105730af9729SIra Weiny 105830af9729SIra Weiny static int cxl_map_regs(struct cxl_mem *cxlm, struct cxl_register_map *map) 105930af9729SIra Weiny { 106030af9729SIra Weiny struct pci_dev *pdev = cxlm->pdev; 106130af9729SIra Weiny struct device *dev = &pdev->dev; 106230af9729SIra Weiny 106330af9729SIra Weiny switch (map->reg_type) { 106408422378SBen Widawsky case CXL_REGLOC_RBI_COMPONENT: 106508422378SBen Widawsky cxl_map_component_regs(pdev, &cxlm->regs.component, map); 106608422378SBen Widawsky dev_dbg(dev, "Mapping component registers...\n"); 106708422378SBen Widawsky break; 106830af9729SIra Weiny case CXL_REGLOC_RBI_MEMDEV: 106930af9729SIra Weiny cxl_map_device_regs(pdev, &cxlm->regs.device_regs, map); 107030af9729SIra Weiny dev_dbg(dev, "Probing device registers...\n"); 107130af9729SIra Weiny break; 107230af9729SIra Weiny default: 107330af9729SIra Weiny break; 107430af9729SIra Weiny } 107530af9729SIra Weiny 107630af9729SIra Weiny return 0; 107730af9729SIra Weiny } 107830af9729SIra Weiny 107907d62eacSIra Weiny static void cxl_decode_register_block(u32 reg_lo, u32 reg_hi, 108007d62eacSIra Weiny u8 *bar, u64 *offset, u8 *reg_type) 108107d62eacSIra Weiny { 108207d62eacSIra Weiny *offset = ((u64)reg_hi << 32) | (reg_lo & CXL_REGLOC_ADDR_MASK); 108307d62eacSIra Weiny *bar = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo); 108407d62eacSIra Weiny *reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); 108507d62eacSIra Weiny } 108607d62eacSIra Weiny 10871d5a4159SBen Widawsky /** 10881d5a4159SBen Widawsky * cxl_mem_setup_regs() - Setup necessary MMIO. 10891d5a4159SBen Widawsky * @cxlm: The CXL memory device to communicate with. 10901d5a4159SBen Widawsky * 10911d5a4159SBen Widawsky * Return: 0 if all necessary registers mapped. 10921d5a4159SBen Widawsky * 10931d5a4159SBen Widawsky * A memory device is required by spec to implement a certain set of MMIO 10941d5a4159SBen Widawsky * regions. The purpose of this function is to enumerate and map those 10951d5a4159SBen Widawsky * registers. 10961d5a4159SBen Widawsky */ 10971d5a4159SBen Widawsky static int cxl_mem_setup_regs(struct cxl_mem *cxlm) 10981d5a4159SBen Widawsky { 10991d5a4159SBen Widawsky struct pci_dev *pdev = cxlm->pdev; 11001d5a4159SBen Widawsky struct device *dev = &pdev->dev; 11011d5a4159SBen Widawsky u32 regloc_size, regblocks; 11026630d31cSBen Widawsky void __iomem *base; 11035b68705dSBen Widawsky int regloc, i, n_maps; 11045b68705dSBen Widawsky struct cxl_register_map *map, maps[CXL_REGLOC_RBI_TYPES]; 110530af9729SIra Weiny int ret = 0; 11061d5a4159SBen Widawsky 11074ad6181eSBen Widawsky regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_DVSEC_ID); 11081d5a4159SBen Widawsky if (!regloc) { 11091d5a4159SBen Widawsky dev_err(dev, "register location dvsec not found\n"); 11101d5a4159SBen Widawsky return -ENXIO; 11111d5a4159SBen Widawsky } 11121d5a4159SBen Widawsky 1113f8a7e8c2SIra Weiny if (pci_request_mem_regions(pdev, pci_name(pdev))) 1114f8a7e8c2SIra Weiny return -ENODEV; 1115f8a7e8c2SIra Weiny 11161d5a4159SBen Widawsky /* Get the size of the Register Locator DVSEC */ 11171d5a4159SBen Widawsky pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size); 11181d5a4159SBen Widawsky regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size); 11191d5a4159SBen Widawsky 11201d5a4159SBen Widawsky regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET; 11211d5a4159SBen Widawsky regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8; 11221d5a4159SBen Widawsky 11235b68705dSBen Widawsky for (i = 0, n_maps = 0; i < regblocks; i++, regloc += 8) { 11241d5a4159SBen Widawsky u32 reg_lo, reg_hi; 11251d5a4159SBen Widawsky u8 reg_type; 112607d62eacSIra Weiny u64 offset; 112707d62eacSIra Weiny u8 bar; 11281d5a4159SBen Widawsky 11291d5a4159SBen Widawsky pci_read_config_dword(pdev, regloc, ®_lo); 11301d5a4159SBen Widawsky pci_read_config_dword(pdev, regloc + 4, ®_hi); 11311d5a4159SBen Widawsky 113207d62eacSIra Weiny cxl_decode_register_block(reg_lo, reg_hi, &bar, &offset, 113307d62eacSIra Weiny ®_type); 113407d62eacSIra Weiny 113507d62eacSIra Weiny dev_dbg(dev, "Found register block in bar %u @ 0x%llx of type %u\n", 113607d62eacSIra Weiny bar, offset, reg_type); 11371d5a4159SBen Widawsky 11381e39db57SBen Widawsky /* Ignore unknown register block types */ 11391e39db57SBen Widawsky if (reg_type > CXL_REGLOC_RBI_MEMDEV) 11401e39db57SBen Widawsky continue; 11411e39db57SBen Widawsky 114207d62eacSIra Weiny base = cxl_mem_map_regblock(cxlm, bar, offset); 11435b68705dSBen Widawsky if (!base) 11445b68705dSBen Widawsky return -ENOMEM; 11451d5a4159SBen Widawsky 11465b68705dSBen Widawsky map = &maps[n_maps]; 114730af9729SIra Weiny map->barno = bar; 114830af9729SIra Weiny map->block_offset = offset; 114930af9729SIra Weiny map->reg_type = reg_type; 115030af9729SIra Weiny 115130af9729SIra Weiny ret = cxl_probe_regs(cxlm, base + offset, map); 115230af9729SIra Weiny 115330af9729SIra Weiny /* Always unmap the regblock regardless of probe success */ 115430af9729SIra Weiny cxl_mem_unmap_regblock(cxlm, base); 115530af9729SIra Weiny 115630af9729SIra Weiny if (ret) 11575b68705dSBen Widawsky return ret; 11585b68705dSBen Widawsky 11595b68705dSBen Widawsky n_maps++; 11601d5a4159SBen Widawsky } 11611d5a4159SBen Widawsky 11629a016527SIra Weiny pci_release_mem_regions(pdev); 11639a016527SIra Weiny 11645b68705dSBen Widawsky for (i = 0; i < n_maps; i++) { 11655b68705dSBen Widawsky ret = cxl_map_regs(cxlm, &maps[i]); 116630af9729SIra Weiny if (ret) 11675b68705dSBen Widawsky break; 116830af9729SIra Weiny } 116930af9729SIra Weiny 117030af9729SIra Weiny return ret; 11711d5a4159SBen Widawsky } 11721d5a4159SBen Widawsky 117321e9f767SBen Widawsky static int cxl_xfer_log(struct cxl_mem *cxlm, uuid_t *uuid, u32 size, u8 *out) 117421e9f767SBen Widawsky { 117521e9f767SBen Widawsky u32 remaining = size; 117621e9f767SBen Widawsky u32 offset = 0; 117721e9f767SBen Widawsky 117821e9f767SBen Widawsky while (remaining) { 117921e9f767SBen Widawsky u32 xfer_size = min_t(u32, remaining, cxlm->payload_size); 118021e9f767SBen Widawsky struct cxl_mbox_get_log { 118121e9f767SBen Widawsky uuid_t uuid; 118221e9f767SBen Widawsky __le32 offset; 118321e9f767SBen Widawsky __le32 length; 118421e9f767SBen Widawsky } __packed log = { 118521e9f767SBen Widawsky .uuid = *uuid, 118621e9f767SBen Widawsky .offset = cpu_to_le32(offset), 118721e9f767SBen Widawsky .length = cpu_to_le32(xfer_size) 118821e9f767SBen Widawsky }; 118921e9f767SBen Widawsky int rc; 119021e9f767SBen Widawsky 119121e9f767SBen Widawsky rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_LOG, &log, 119221e9f767SBen Widawsky sizeof(log), out, xfer_size); 119321e9f767SBen Widawsky if (rc < 0) 119421e9f767SBen Widawsky return rc; 119521e9f767SBen Widawsky 119621e9f767SBen Widawsky out += xfer_size; 119721e9f767SBen Widawsky remaining -= xfer_size; 119821e9f767SBen Widawsky offset += xfer_size; 119921e9f767SBen Widawsky } 120021e9f767SBen Widawsky 120121e9f767SBen Widawsky return 0; 120221e9f767SBen Widawsky } 120321e9f767SBen Widawsky 120421e9f767SBen Widawsky /** 120521e9f767SBen Widawsky * cxl_walk_cel() - Walk through the Command Effects Log. 120621e9f767SBen Widawsky * @cxlm: Device. 120721e9f767SBen Widawsky * @size: Length of the Command Effects Log. 120821e9f767SBen Widawsky * @cel: CEL 120921e9f767SBen Widawsky * 121021e9f767SBen Widawsky * Iterate over each entry in the CEL and determine if the driver supports the 121121e9f767SBen Widawsky * command. If so, the command is enabled for the device and can be used later. 121221e9f767SBen Widawsky */ 121321e9f767SBen Widawsky static void cxl_walk_cel(struct cxl_mem *cxlm, size_t size, u8 *cel) 121421e9f767SBen Widawsky { 121521e9f767SBen Widawsky struct cel_entry { 121621e9f767SBen Widawsky __le16 opcode; 121721e9f767SBen Widawsky __le16 effect; 121821e9f767SBen Widawsky } __packed * cel_entry; 121921e9f767SBen Widawsky const int cel_entries = size / sizeof(*cel_entry); 122021e9f767SBen Widawsky int i; 122121e9f767SBen Widawsky 122221e9f767SBen Widawsky cel_entry = (struct cel_entry *)cel; 122321e9f767SBen Widawsky 122421e9f767SBen Widawsky for (i = 0; i < cel_entries; i++) { 122521e9f767SBen Widawsky u16 opcode = le16_to_cpu(cel_entry[i].opcode); 122621e9f767SBen Widawsky struct cxl_mem_command *cmd = cxl_mem_find_command(opcode); 122721e9f767SBen Widawsky 122821e9f767SBen Widawsky if (!cmd) { 122921e9f767SBen Widawsky dev_dbg(&cxlm->pdev->dev, 123021e9f767SBen Widawsky "Opcode 0x%04x unsupported by driver", opcode); 123121e9f767SBen Widawsky continue; 123221e9f767SBen Widawsky } 123321e9f767SBen Widawsky 123421e9f767SBen Widawsky set_bit(cmd->info.id, cxlm->enabled_cmds); 123521e9f767SBen Widawsky } 123621e9f767SBen Widawsky } 123721e9f767SBen Widawsky 123821e9f767SBen Widawsky struct cxl_mbox_get_supported_logs { 123921e9f767SBen Widawsky __le16 entries; 124021e9f767SBen Widawsky u8 rsvd[6]; 124121e9f767SBen Widawsky struct gsl_entry { 124221e9f767SBen Widawsky uuid_t uuid; 124321e9f767SBen Widawsky __le32 size; 124421e9f767SBen Widawsky } __packed entry[]; 124521e9f767SBen Widawsky } __packed; 124621e9f767SBen Widawsky 124721e9f767SBen Widawsky static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_mem *cxlm) 124821e9f767SBen Widawsky { 124921e9f767SBen Widawsky struct cxl_mbox_get_supported_logs *ret; 125021e9f767SBen Widawsky int rc; 125121e9f767SBen Widawsky 125221e9f767SBen Widawsky ret = kvmalloc(cxlm->payload_size, GFP_KERNEL); 125321e9f767SBen Widawsky if (!ret) 125421e9f767SBen Widawsky return ERR_PTR(-ENOMEM); 125521e9f767SBen Widawsky 125621e9f767SBen Widawsky rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_SUPPORTED_LOGS, NULL, 125721e9f767SBen Widawsky 0, ret, cxlm->payload_size); 125821e9f767SBen Widawsky if (rc < 0) { 125921e9f767SBen Widawsky kvfree(ret); 126021e9f767SBen Widawsky return ERR_PTR(rc); 126121e9f767SBen Widawsky } 126221e9f767SBen Widawsky 126321e9f767SBen Widawsky return ret; 126421e9f767SBen Widawsky } 126521e9f767SBen Widawsky 126621e9f767SBen Widawsky /** 1267f847502aSIra Weiny * cxl_mem_get_partition_info - Get partition info 1268f847502aSIra Weiny * @cxlm: The device to act on 1269f847502aSIra Weiny * @active_volatile_bytes: returned active volatile capacity 1270f847502aSIra Weiny * @active_persistent_bytes: returned active persistent capacity 1271f847502aSIra Weiny * @next_volatile_bytes: return next volatile capacity 1272f847502aSIra Weiny * @next_persistent_bytes: return next persistent capacity 1273f847502aSIra Weiny * 1274f847502aSIra Weiny * Retrieve the current partition info for the device specified. If not 0, the 1275f847502aSIra Weiny * 'next' values are pending and take affect on next cold reset. 1276f847502aSIra Weiny * 1277f847502aSIra Weiny * Return: 0 if no error: or the result of the mailbox command. 1278f847502aSIra Weiny * 1279f847502aSIra Weiny * See CXL @8.2.9.5.2.1 Get Partition Info 1280f847502aSIra Weiny */ 1281f847502aSIra Weiny static int cxl_mem_get_partition_info(struct cxl_mem *cxlm, 1282f847502aSIra Weiny u64 *active_volatile_bytes, 1283f847502aSIra Weiny u64 *active_persistent_bytes, 1284f847502aSIra Weiny u64 *next_volatile_bytes, 1285f847502aSIra Weiny u64 *next_persistent_bytes) 1286f847502aSIra Weiny { 1287f847502aSIra Weiny struct cxl_mbox_get_partition_info { 1288f847502aSIra Weiny __le64 active_volatile_cap; 1289f847502aSIra Weiny __le64 active_persistent_cap; 1290f847502aSIra Weiny __le64 next_volatile_cap; 1291f847502aSIra Weiny __le64 next_persistent_cap; 1292f847502aSIra Weiny } __packed pi; 1293f847502aSIra Weiny int rc; 1294f847502aSIra Weiny 1295f847502aSIra Weiny rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_PARTITION_INFO, 1296f847502aSIra Weiny NULL, 0, &pi, sizeof(pi)); 1297f847502aSIra Weiny if (rc) 1298f847502aSIra Weiny return rc; 1299f847502aSIra Weiny 1300f847502aSIra Weiny *active_volatile_bytes = le64_to_cpu(pi.active_volatile_cap); 1301f847502aSIra Weiny *active_persistent_bytes = le64_to_cpu(pi.active_persistent_cap); 1302f847502aSIra Weiny *next_volatile_bytes = le64_to_cpu(pi.next_volatile_cap); 1303f847502aSIra Weiny *next_persistent_bytes = le64_to_cpu(pi.next_volatile_cap); 1304f847502aSIra Weiny 1305f847502aSIra Weiny *active_volatile_bytes *= CXL_CAPACITY_MULTIPLIER; 1306f847502aSIra Weiny *active_persistent_bytes *= CXL_CAPACITY_MULTIPLIER; 1307f847502aSIra Weiny *next_volatile_bytes *= CXL_CAPACITY_MULTIPLIER; 1308f847502aSIra Weiny *next_persistent_bytes *= CXL_CAPACITY_MULTIPLIER; 1309f847502aSIra Weiny 1310f847502aSIra Weiny return 0; 1311f847502aSIra Weiny } 1312f847502aSIra Weiny 1313f847502aSIra Weiny /** 131421e9f767SBen Widawsky * cxl_mem_enumerate_cmds() - Enumerate commands for a device. 131521e9f767SBen Widawsky * @cxlm: The device. 131621e9f767SBen Widawsky * 131721e9f767SBen Widawsky * Returns 0 if enumerate completed successfully. 131821e9f767SBen Widawsky * 131921e9f767SBen Widawsky * CXL devices have optional support for certain commands. This function will 132021e9f767SBen Widawsky * determine the set of supported commands for the hardware and update the 132121e9f767SBen Widawsky * enabled_cmds bitmap in the @cxlm. 132221e9f767SBen Widawsky */ 132321e9f767SBen Widawsky static int cxl_mem_enumerate_cmds(struct cxl_mem *cxlm) 132421e9f767SBen Widawsky { 132521e9f767SBen Widawsky struct cxl_mbox_get_supported_logs *gsl; 132621e9f767SBen Widawsky struct device *dev = &cxlm->pdev->dev; 132721e9f767SBen Widawsky struct cxl_mem_command *cmd; 132821e9f767SBen Widawsky int i, rc; 132921e9f767SBen Widawsky 133021e9f767SBen Widawsky gsl = cxl_get_gsl(cxlm); 133121e9f767SBen Widawsky if (IS_ERR(gsl)) 133221e9f767SBen Widawsky return PTR_ERR(gsl); 133321e9f767SBen Widawsky 133421e9f767SBen Widawsky rc = -ENOENT; 133521e9f767SBen Widawsky for (i = 0; i < le16_to_cpu(gsl->entries); i++) { 133621e9f767SBen Widawsky u32 size = le32_to_cpu(gsl->entry[i].size); 133721e9f767SBen Widawsky uuid_t uuid = gsl->entry[i].uuid; 133821e9f767SBen Widawsky u8 *log; 133921e9f767SBen Widawsky 134021e9f767SBen Widawsky dev_dbg(dev, "Found LOG type %pU of size %d", &uuid, size); 134121e9f767SBen Widawsky 134221e9f767SBen Widawsky if (!uuid_equal(&uuid, &log_uuid[CEL_UUID])) 134321e9f767SBen Widawsky continue; 134421e9f767SBen Widawsky 134521e9f767SBen Widawsky log = kvmalloc(size, GFP_KERNEL); 134621e9f767SBen Widawsky if (!log) { 134721e9f767SBen Widawsky rc = -ENOMEM; 134821e9f767SBen Widawsky goto out; 134921e9f767SBen Widawsky } 135021e9f767SBen Widawsky 135121e9f767SBen Widawsky rc = cxl_xfer_log(cxlm, &uuid, size, log); 135221e9f767SBen Widawsky if (rc) { 135321e9f767SBen Widawsky kvfree(log); 135421e9f767SBen Widawsky goto out; 135521e9f767SBen Widawsky } 135621e9f767SBen Widawsky 135721e9f767SBen Widawsky cxl_walk_cel(cxlm, size, log); 135821e9f767SBen Widawsky kvfree(log); 135921e9f767SBen Widawsky 136021e9f767SBen Widawsky /* In case CEL was bogus, enable some default commands. */ 136121e9f767SBen Widawsky cxl_for_each_cmd(cmd) 136221e9f767SBen Widawsky if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE) 136321e9f767SBen Widawsky set_bit(cmd->info.id, cxlm->enabled_cmds); 136421e9f767SBen Widawsky 136521e9f767SBen Widawsky /* Found the required CEL */ 136621e9f767SBen Widawsky rc = 0; 136721e9f767SBen Widawsky } 136821e9f767SBen Widawsky 136921e9f767SBen Widawsky out: 137021e9f767SBen Widawsky kvfree(gsl); 137121e9f767SBen Widawsky return rc; 137221e9f767SBen Widawsky } 137321e9f767SBen Widawsky 137421e9f767SBen Widawsky /** 137521e9f767SBen Widawsky * cxl_mem_identify() - Send the IDENTIFY command to the device. 137621e9f767SBen Widawsky * @cxlm: The device to identify. 137721e9f767SBen Widawsky * 137821e9f767SBen Widawsky * Return: 0 if identify was executed successfully. 137921e9f767SBen Widawsky * 138021e9f767SBen Widawsky * This will dispatch the identify command to the device and on success populate 138121e9f767SBen Widawsky * structures to be exported to sysfs. 138221e9f767SBen Widawsky */ 138321e9f767SBen Widawsky static int cxl_mem_identify(struct cxl_mem *cxlm) 138421e9f767SBen Widawsky { 138521e9f767SBen Widawsky /* See CXL 2.0 Table 175 Identify Memory Device Output Payload */ 138621e9f767SBen Widawsky struct cxl_mbox_identify { 138721e9f767SBen Widawsky char fw_revision[0x10]; 138821e9f767SBen Widawsky __le64 total_capacity; 138921e9f767SBen Widawsky __le64 volatile_capacity; 139021e9f767SBen Widawsky __le64 persistent_capacity; 139121e9f767SBen Widawsky __le64 partition_align; 139221e9f767SBen Widawsky __le16 info_event_log_size; 139321e9f767SBen Widawsky __le16 warning_event_log_size; 139421e9f767SBen Widawsky __le16 failure_event_log_size; 139521e9f767SBen Widawsky __le16 fatal_event_log_size; 139621e9f767SBen Widawsky __le32 lsa_size; 139721e9f767SBen Widawsky u8 poison_list_max_mer[3]; 139821e9f767SBen Widawsky __le16 inject_poison_limit; 139921e9f767SBen Widawsky u8 poison_caps; 140021e9f767SBen Widawsky u8 qos_telemetry_caps; 140121e9f767SBen Widawsky } __packed id; 140221e9f767SBen Widawsky int rc; 140321e9f767SBen Widawsky 140421e9f767SBen Widawsky rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_IDENTIFY, NULL, 0, &id, 140521e9f767SBen Widawsky sizeof(id)); 140621e9f767SBen Widawsky if (rc < 0) 140721e9f767SBen Widawsky return rc; 140821e9f767SBen Widawsky 14090b9159d0SIra Weiny cxlm->total_bytes = le64_to_cpu(id.total_capacity); 14100b9159d0SIra Weiny cxlm->total_bytes *= CXL_CAPACITY_MULTIPLIER; 14110b9159d0SIra Weiny 14120b9159d0SIra Weiny cxlm->volatile_only_bytes = le64_to_cpu(id.volatile_capacity); 14130b9159d0SIra Weiny cxlm->volatile_only_bytes *= CXL_CAPACITY_MULTIPLIER; 14140b9159d0SIra Weiny 14150b9159d0SIra Weiny cxlm->persistent_only_bytes = le64_to_cpu(id.persistent_capacity); 14160b9159d0SIra Weiny cxlm->persistent_only_bytes *= CXL_CAPACITY_MULTIPLIER; 14170b9159d0SIra Weiny 14180b9159d0SIra Weiny cxlm->partition_align_bytes = le64_to_cpu(id.partition_align); 14190b9159d0SIra Weiny cxlm->partition_align_bytes *= CXL_CAPACITY_MULTIPLIER; 14200b9159d0SIra Weiny 14210b9159d0SIra Weiny dev_dbg(&cxlm->pdev->dev, "Identify Memory Device\n" 14220b9159d0SIra Weiny " total_bytes = %#llx\n" 14230b9159d0SIra Weiny " volatile_only_bytes = %#llx\n" 14240b9159d0SIra Weiny " persistent_only_bytes = %#llx\n" 14250b9159d0SIra Weiny " partition_align_bytes = %#llx\n", 14260b9159d0SIra Weiny cxlm->total_bytes, 14270b9159d0SIra Weiny cxlm->volatile_only_bytes, 14280b9159d0SIra Weiny cxlm->persistent_only_bytes, 14290b9159d0SIra Weiny cxlm->partition_align_bytes); 14300b9159d0SIra Weiny 1431f847502aSIra Weiny cxlm->lsa_size = le32_to_cpu(id.lsa_size); 1432f847502aSIra Weiny memcpy(cxlm->firmware_version, id.fw_revision, sizeof(id.fw_revision)); 1433f847502aSIra Weiny 1434f847502aSIra Weiny return 0; 1435f847502aSIra Weiny } 1436f847502aSIra Weiny 1437f847502aSIra Weiny static int cxl_mem_create_range_info(struct cxl_mem *cxlm) 1438f847502aSIra Weiny { 1439f847502aSIra Weiny int rc; 1440f847502aSIra Weiny 1441f847502aSIra Weiny if (cxlm->partition_align_bytes == 0) { 1442f847502aSIra Weiny cxlm->ram_range.start = 0; 1443f847502aSIra Weiny cxlm->ram_range.end = cxlm->volatile_only_bytes - 1; 1444ceeb0da0SIra Weiny cxlm->pmem_range.start = cxlm->volatile_only_bytes; 1445ceeb0da0SIra Weiny cxlm->pmem_range.end = cxlm->volatile_only_bytes + 1446ceeb0da0SIra Weiny cxlm->persistent_only_bytes - 1; 1447f847502aSIra Weiny return 0; 1448f847502aSIra Weiny } 1449f847502aSIra Weiny 1450f847502aSIra Weiny rc = cxl_mem_get_partition_info(cxlm, 1451f847502aSIra Weiny &cxlm->active_volatile_bytes, 1452f847502aSIra Weiny &cxlm->active_persistent_bytes, 1453f847502aSIra Weiny &cxlm->next_volatile_bytes, 1454f847502aSIra Weiny &cxlm->next_persistent_bytes); 1455f847502aSIra Weiny if (rc < 0) { 1456f847502aSIra Weiny dev_err(&cxlm->pdev->dev, "Failed to query partition information\n"); 1457f847502aSIra Weiny return rc; 1458f847502aSIra Weiny } 1459f847502aSIra Weiny 1460f847502aSIra Weiny dev_dbg(&cxlm->pdev->dev, "Get Partition Info\n" 1461f847502aSIra Weiny " active_volatile_bytes = %#llx\n" 1462f847502aSIra Weiny " active_persistent_bytes = %#llx\n" 1463f847502aSIra Weiny " next_volatile_bytes = %#llx\n" 1464f847502aSIra Weiny " next_persistent_bytes = %#llx\n", 1465f847502aSIra Weiny cxlm->active_volatile_bytes, 1466f847502aSIra Weiny cxlm->active_persistent_bytes, 1467f847502aSIra Weiny cxlm->next_volatile_bytes, 1468f847502aSIra Weiny cxlm->next_persistent_bytes); 1469f847502aSIra Weiny 147021e9f767SBen Widawsky cxlm->ram_range.start = 0; 1471f847502aSIra Weiny cxlm->ram_range.end = cxlm->active_volatile_bytes - 1; 147221e9f767SBen Widawsky 1473ceeb0da0SIra Weiny cxlm->pmem_range.start = cxlm->active_volatile_bytes; 1474ceeb0da0SIra Weiny cxlm->pmem_range.end = cxlm->active_volatile_bytes + 1475ceeb0da0SIra Weiny cxlm->active_persistent_bytes - 1; 147621e9f767SBen Widawsky 147721e9f767SBen Widawsky return 0; 147821e9f767SBen Widawsky } 147921e9f767SBen Widawsky 148021e9f767SBen Widawsky static int cxl_mem_probe(struct pci_dev *pdev, const struct pci_device_id *id) 148121e9f767SBen Widawsky { 148221083f51SDan Williams struct cxl_memdev *cxlmd; 14831b0a1a2aSBen Widawsky struct cxl_mem *cxlm; 14841d5a4159SBen Widawsky int rc; 148521e9f767SBen Widawsky 148621e9f767SBen Widawsky rc = pcim_enable_device(pdev); 148721e9f767SBen Widawsky if (rc) 148821e9f767SBen Widawsky return rc; 148921e9f767SBen Widawsky 14901b0a1a2aSBen Widawsky cxlm = cxl_mem_create(pdev); 14911b0a1a2aSBen Widawsky if (IS_ERR(cxlm)) 14921b0a1a2aSBen Widawsky return PTR_ERR(cxlm); 14931b0a1a2aSBen Widawsky 149421e9f767SBen Widawsky rc = cxl_mem_setup_regs(cxlm); 149521e9f767SBen Widawsky if (rc) 149621e9f767SBen Widawsky return rc; 149721e9f767SBen Widawsky 149821e9f767SBen Widawsky rc = cxl_mem_setup_mailbox(cxlm); 149921e9f767SBen Widawsky if (rc) 150021e9f767SBen Widawsky return rc; 150121e9f767SBen Widawsky 150221e9f767SBen Widawsky rc = cxl_mem_enumerate_cmds(cxlm); 150321e9f767SBen Widawsky if (rc) 150421e9f767SBen Widawsky return rc; 150521e9f767SBen Widawsky 150621e9f767SBen Widawsky rc = cxl_mem_identify(cxlm); 150721e9f767SBen Widawsky if (rc) 150821e9f767SBen Widawsky return rc; 150921e9f767SBen Widawsky 1510f847502aSIra Weiny rc = cxl_mem_create_range_info(cxlm); 1511f847502aSIra Weiny if (rc) 1512f847502aSIra Weiny return rc; 1513f847502aSIra Weiny 15149cc238c7SDan Williams cxlmd = devm_cxl_add_memdev(&pdev->dev, cxlm, &cxl_memdev_fops); 151521083f51SDan Williams if (IS_ERR(cxlmd)) 151621083f51SDan Williams return PTR_ERR(cxlmd); 151721083f51SDan Williams 151821083f51SDan Williams if (range_len(&cxlm->pmem_range) && IS_ENABLED(CONFIG_CXL_PMEM)) 151921083f51SDan Williams rc = devm_cxl_add_nvdimm(&pdev->dev, cxlmd); 152021083f51SDan Williams 152121083f51SDan Williams return rc; 152221e9f767SBen Widawsky } 152321e9f767SBen Widawsky 152421e9f767SBen Widawsky static const struct pci_device_id cxl_mem_pci_tbl[] = { 152521e9f767SBen Widawsky /* PCI class code for CXL.mem Type-3 Devices */ 152621e9f767SBen Widawsky { PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)}, 152721e9f767SBen Widawsky { /* terminate list */ }, 152821e9f767SBen Widawsky }; 152921e9f767SBen Widawsky MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl); 153021e9f767SBen Widawsky 153121e9f767SBen Widawsky static struct pci_driver cxl_mem_driver = { 153221e9f767SBen Widawsky .name = KBUILD_MODNAME, 153321e9f767SBen Widawsky .id_table = cxl_mem_pci_tbl, 153421e9f767SBen Widawsky .probe = cxl_mem_probe, 153521e9f767SBen Widawsky .driver = { 153621e9f767SBen Widawsky .probe_type = PROBE_PREFER_ASYNCHRONOUS, 153721e9f767SBen Widawsky }, 153821e9f767SBen Widawsky }; 153921e9f767SBen Widawsky 154021e9f767SBen Widawsky static __init int cxl_mem_init(void) 154121e9f767SBen Widawsky { 154221e9f767SBen Widawsky struct dentry *mbox_debugfs; 154321e9f767SBen Widawsky int rc; 154421e9f767SBen Widawsky 154521e9f767SBen Widawsky /* Double check the anonymous union trickery in struct cxl_regs */ 154621e9f767SBen Widawsky BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) != 154721e9f767SBen Widawsky offsetof(struct cxl_regs, device_regs.memdev)); 154821e9f767SBen Widawsky 15493d135db5SBen Widawsky rc = pci_register_driver(&cxl_mem_driver); 155021e9f767SBen Widawsky if (rc) 155121e9f767SBen Widawsky return rc; 155221e9f767SBen Widawsky 155321e9f767SBen Widawsky cxl_debugfs = debugfs_create_dir("cxl", NULL); 155421e9f767SBen Widawsky mbox_debugfs = debugfs_create_dir("mbox", cxl_debugfs); 155521e9f767SBen Widawsky debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs, 155621e9f767SBen Widawsky &cxl_raw_allow_all); 155721e9f767SBen Widawsky 155821e9f767SBen Widawsky return 0; 155921e9f767SBen Widawsky } 156021e9f767SBen Widawsky 156121e9f767SBen Widawsky static __exit void cxl_mem_exit(void) 156221e9f767SBen Widawsky { 156321e9f767SBen Widawsky debugfs_remove_recursive(cxl_debugfs); 156421e9f767SBen Widawsky pci_unregister_driver(&cxl_mem_driver); 156521e9f767SBen Widawsky } 156621e9f767SBen Widawsky 156721e9f767SBen Widawsky MODULE_LICENSE("GPL v2"); 156821e9f767SBen Widawsky module_init(cxl_mem_init); 156921e9f767SBen Widawsky module_exit(cxl_mem_exit); 157021e9f767SBen Widawsky MODULE_IMPORT_NS(CXL); 1571