1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2020 Intel Corporation. All rights reserved. */
3 #include <linux/security.h>
4 #include <linux/debugfs.h>
5 #include <linux/ktime.h>
6 #include <linux/mutex.h>
7 #include <asm/unaligned.h>
8 #include <cxlpci.h>
9 #include <cxlmem.h>
10 #include <cxl.h>
11
12 #include "core.h"
13 #include "trace.h"
14
15 static bool cxl_raw_allow_all;
16
17 /**
18 * DOC: cxl mbox
19 *
20 * Core implementation of the CXL 2.0 Type-3 Memory Device Mailbox. The
21 * implementation is used by the cxl_pci driver to initialize the device
22 * and implement the cxl_mem.h IOCTL UAPI. It also implements the
23 * backend of the cxl_pmem_ctl() transport for LIBNVDIMM.
24 */
25
26 #define cxl_for_each_cmd(cmd) \
27 for ((cmd) = &cxl_mem_commands[0]; \
28 ((cmd) - cxl_mem_commands) < ARRAY_SIZE(cxl_mem_commands); (cmd)++)
29
30 #define CXL_CMD(_id, sin, sout, _flags) \
31 [CXL_MEM_COMMAND_ID_##_id] = { \
32 .info = { \
33 .id = CXL_MEM_COMMAND_ID_##_id, \
34 .size_in = sin, \
35 .size_out = sout, \
36 }, \
37 .opcode = CXL_MBOX_OP_##_id, \
38 .flags = _flags, \
39 }
40
41 #define CXL_VARIABLE_PAYLOAD ~0U
42 /*
43 * This table defines the supported mailbox commands for the driver. This table
44 * is made up of a UAPI structure. Non-negative values as parameters in the
45 * table will be validated against the user's input. For example, if size_in is
46 * 0, and the user passed in 1, it is an error.
47 */
48 static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
49 CXL_CMD(IDENTIFY, 0, 0x43, CXL_CMD_FLAG_FORCE_ENABLE),
50 #ifdef CONFIG_CXL_MEM_RAW_COMMANDS
51 CXL_CMD(RAW, CXL_VARIABLE_PAYLOAD, CXL_VARIABLE_PAYLOAD, 0),
52 #endif
53 CXL_CMD(GET_SUPPORTED_LOGS, 0, CXL_VARIABLE_PAYLOAD, CXL_CMD_FLAG_FORCE_ENABLE),
54 CXL_CMD(GET_FW_INFO, 0, 0x50, 0),
55 CXL_CMD(GET_PARTITION_INFO, 0, 0x20, 0),
56 CXL_CMD(GET_LSA, 0x8, CXL_VARIABLE_PAYLOAD, 0),
57 CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0),
58 CXL_CMD(GET_LOG, 0x18, CXL_VARIABLE_PAYLOAD, CXL_CMD_FLAG_FORCE_ENABLE),
59 CXL_CMD(SET_PARTITION_INFO, 0x0a, 0, 0),
60 CXL_CMD(SET_LSA, CXL_VARIABLE_PAYLOAD, 0, 0),
61 CXL_CMD(GET_ALERT_CONFIG, 0, 0x10, 0),
62 CXL_CMD(SET_ALERT_CONFIG, 0xc, 0, 0),
63 CXL_CMD(GET_SHUTDOWN_STATE, 0, 0x1, 0),
64 CXL_CMD(SET_SHUTDOWN_STATE, 0x1, 0, 0),
65 CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0),
66 };
67
68 /*
69 * Commands that RAW doesn't permit. The rationale for each:
70 *
71 * CXL_MBOX_OP_ACTIVATE_FW: Firmware activation requires adjustment /
72 * coordination of transaction timeout values at the root bridge level.
73 *
74 * CXL_MBOX_OP_SET_PARTITION_INFO: The device memory map may change live
75 * and needs to be coordinated with HDM updates.
76 *
77 * CXL_MBOX_OP_SET_LSA: The label storage area may be cached by the
78 * driver and any writes from userspace invalidates those contents.
79 *
80 * CXL_MBOX_OP_SET_SHUTDOWN_STATE: Set shutdown state assumes no writes
81 * to the device after it is marked clean, userspace can not make that
82 * assertion.
83 *
84 * CXL_MBOX_OP_[GET_]SCAN_MEDIA: The kernel provides a native error list that
85 * is kept up to date with patrol notifications and error management.
86 *
87 * CXL_MBOX_OP_[GET_,INJECT_,CLEAR_]POISON: These commands require kernel
88 * driver orchestration for safety.
89 */
90 static u16 cxl_disabled_raw_commands[] = {
91 CXL_MBOX_OP_ACTIVATE_FW,
92 CXL_MBOX_OP_SET_PARTITION_INFO,
93 CXL_MBOX_OP_SET_LSA,
94 CXL_MBOX_OP_SET_SHUTDOWN_STATE,
95 CXL_MBOX_OP_SCAN_MEDIA,
96 CXL_MBOX_OP_GET_SCAN_MEDIA,
97 CXL_MBOX_OP_GET_POISON,
98 CXL_MBOX_OP_INJECT_POISON,
99 CXL_MBOX_OP_CLEAR_POISON,
100 };
101
102 /*
103 * Command sets that RAW doesn't permit. All opcodes in this set are
104 * disabled because they pass plain text security payloads over the
105 * user/kernel boundary. This functionality is intended to be wrapped
106 * behind the keys ABI which allows for encrypted payloads in the UAPI
107 */
108 static u8 security_command_sets[] = {
109 0x44, /* Sanitize */
110 0x45, /* Persistent Memory Data-at-rest Security */
111 0x46, /* Security Passthrough */
112 };
113
cxl_is_security_command(u16 opcode)114 static bool cxl_is_security_command(u16 opcode)
115 {
116 int i;
117
118 for (i = 0; i < ARRAY_SIZE(security_command_sets); i++)
119 if (security_command_sets[i] == (opcode >> 8))
120 return true;
121 return false;
122 }
123
cxl_set_security_cmd_enabled(struct cxl_security_state * security,u16 opcode)124 static void cxl_set_security_cmd_enabled(struct cxl_security_state *security,
125 u16 opcode)
126 {
127 switch (opcode) {
128 case CXL_MBOX_OP_SANITIZE:
129 set_bit(CXL_SEC_ENABLED_SANITIZE, security->enabled_cmds);
130 break;
131 case CXL_MBOX_OP_SECURE_ERASE:
132 set_bit(CXL_SEC_ENABLED_SECURE_ERASE,
133 security->enabled_cmds);
134 break;
135 case CXL_MBOX_OP_GET_SECURITY_STATE:
136 set_bit(CXL_SEC_ENABLED_GET_SECURITY_STATE,
137 security->enabled_cmds);
138 break;
139 case CXL_MBOX_OP_SET_PASSPHRASE:
140 set_bit(CXL_SEC_ENABLED_SET_PASSPHRASE,
141 security->enabled_cmds);
142 break;
143 case CXL_MBOX_OP_DISABLE_PASSPHRASE:
144 set_bit(CXL_SEC_ENABLED_DISABLE_PASSPHRASE,
145 security->enabled_cmds);
146 break;
147 case CXL_MBOX_OP_UNLOCK:
148 set_bit(CXL_SEC_ENABLED_UNLOCK, security->enabled_cmds);
149 break;
150 case CXL_MBOX_OP_FREEZE_SECURITY:
151 set_bit(CXL_SEC_ENABLED_FREEZE_SECURITY,
152 security->enabled_cmds);
153 break;
154 case CXL_MBOX_OP_PASSPHRASE_SECURE_ERASE:
155 set_bit(CXL_SEC_ENABLED_PASSPHRASE_SECURE_ERASE,
156 security->enabled_cmds);
157 break;
158 default:
159 break;
160 }
161 }
162
cxl_is_poison_command(u16 opcode)163 static bool cxl_is_poison_command(u16 opcode)
164 {
165 #define CXL_MBOX_OP_POISON_CMDS 0x43
166
167 if ((opcode >> 8) == CXL_MBOX_OP_POISON_CMDS)
168 return true;
169
170 return false;
171 }
172
cxl_set_poison_cmd_enabled(struct cxl_poison_state * poison,u16 opcode)173 static void cxl_set_poison_cmd_enabled(struct cxl_poison_state *poison,
174 u16 opcode)
175 {
176 switch (opcode) {
177 case CXL_MBOX_OP_GET_POISON:
178 set_bit(CXL_POISON_ENABLED_LIST, poison->enabled_cmds);
179 break;
180 case CXL_MBOX_OP_INJECT_POISON:
181 set_bit(CXL_POISON_ENABLED_INJECT, poison->enabled_cmds);
182 break;
183 case CXL_MBOX_OP_CLEAR_POISON:
184 set_bit(CXL_POISON_ENABLED_CLEAR, poison->enabled_cmds);
185 break;
186 case CXL_MBOX_OP_GET_SCAN_MEDIA_CAPS:
187 set_bit(CXL_POISON_ENABLED_SCAN_CAPS, poison->enabled_cmds);
188 break;
189 case CXL_MBOX_OP_SCAN_MEDIA:
190 set_bit(CXL_POISON_ENABLED_SCAN_MEDIA, poison->enabled_cmds);
191 break;
192 case CXL_MBOX_OP_GET_SCAN_MEDIA:
193 set_bit(CXL_POISON_ENABLED_SCAN_RESULTS, poison->enabled_cmds);
194 break;
195 default:
196 break;
197 }
198 }
199
cxl_mem_find_command(u16 opcode)200 static struct cxl_mem_command *cxl_mem_find_command(u16 opcode)
201 {
202 struct cxl_mem_command *c;
203
204 cxl_for_each_cmd(c)
205 if (c->opcode == opcode)
206 return c;
207
208 return NULL;
209 }
210
cxl_mem_opcode_to_name(u16 opcode)211 static const char *cxl_mem_opcode_to_name(u16 opcode)
212 {
213 struct cxl_mem_command *c;
214
215 c = cxl_mem_find_command(opcode);
216 if (!c)
217 return NULL;
218
219 return cxl_command_names[c->info.id].name;
220 }
221
222 /**
223 * cxl_internal_send_cmd() - Kernel internal interface to send a mailbox command
224 * @mds: The driver data for the operation
225 * @mbox_cmd: initialized command to execute
226 *
227 * Context: Any context.
228 * Return:
229 * * %>=0 - Number of bytes returned in @out.
230 * * %-E2BIG - Payload is too large for hardware.
231 * * %-EBUSY - Couldn't acquire exclusive mailbox access.
232 * * %-EFAULT - Hardware error occurred.
233 * * %-ENXIO - Command completed, but device reported an error.
234 * * %-EIO - Unexpected output size.
235 *
236 * Mailbox commands may execute successfully yet the device itself reported an
237 * error. While this distinction can be useful for commands from userspace, the
238 * kernel will only be able to use results when both are successful.
239 */
cxl_internal_send_cmd(struct cxl_memdev_state * mds,struct cxl_mbox_cmd * mbox_cmd)240 int cxl_internal_send_cmd(struct cxl_memdev_state *mds,
241 struct cxl_mbox_cmd *mbox_cmd)
242 {
243 size_t out_size, min_out;
244 int rc;
245
246 if (mbox_cmd->size_in > mds->payload_size ||
247 mbox_cmd->size_out > mds->payload_size)
248 return -E2BIG;
249
250 out_size = mbox_cmd->size_out;
251 min_out = mbox_cmd->min_out;
252 rc = mds->mbox_send(mds, mbox_cmd);
253 /*
254 * EIO is reserved for a payload size mismatch and mbox_send()
255 * may not return this error.
256 */
257 if (WARN_ONCE(rc == -EIO, "Bad return code: -EIO"))
258 return -ENXIO;
259 if (rc)
260 return rc;
261
262 if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS &&
263 mbox_cmd->return_code != CXL_MBOX_CMD_RC_BACKGROUND)
264 return cxl_mbox_cmd_rc2errno(mbox_cmd);
265
266 if (!out_size)
267 return 0;
268
269 /*
270 * Variable sized output needs to at least satisfy the caller's
271 * minimum if not the fully requested size.
272 */
273 if (min_out == 0)
274 min_out = out_size;
275
276 if (mbox_cmd->size_out < min_out)
277 return -EIO;
278 return 0;
279 }
280 EXPORT_SYMBOL_NS_GPL(cxl_internal_send_cmd, CXL);
281
cxl_mem_raw_command_allowed(u16 opcode)282 static bool cxl_mem_raw_command_allowed(u16 opcode)
283 {
284 int i;
285
286 if (!IS_ENABLED(CONFIG_CXL_MEM_RAW_COMMANDS))
287 return false;
288
289 if (security_locked_down(LOCKDOWN_PCI_ACCESS))
290 return false;
291
292 if (cxl_raw_allow_all)
293 return true;
294
295 if (cxl_is_security_command(opcode))
296 return false;
297
298 for (i = 0; i < ARRAY_SIZE(cxl_disabled_raw_commands); i++)
299 if (cxl_disabled_raw_commands[i] == opcode)
300 return false;
301
302 return true;
303 }
304
305 /**
306 * cxl_payload_from_user_allowed() - Check contents of in_payload.
307 * @opcode: The mailbox command opcode.
308 * @payload_in: Pointer to the input payload passed in from user space.
309 *
310 * Return:
311 * * true - payload_in passes check for @opcode.
312 * * false - payload_in contains invalid or unsupported values.
313 *
314 * The driver may inspect payload contents before sending a mailbox
315 * command from user space to the device. The intent is to reject
316 * commands with input payloads that are known to be unsafe. This
317 * check is not intended to replace the users careful selection of
318 * mailbox command parameters and makes no guarantee that the user
319 * command will succeed, nor that it is appropriate.
320 *
321 * The specific checks are determined by the opcode.
322 */
cxl_payload_from_user_allowed(u16 opcode,void * payload_in)323 static bool cxl_payload_from_user_allowed(u16 opcode, void *payload_in)
324 {
325 switch (opcode) {
326 case CXL_MBOX_OP_SET_PARTITION_INFO: {
327 struct cxl_mbox_set_partition_info *pi = payload_in;
328
329 if (pi->flags & CXL_SET_PARTITION_IMMEDIATE_FLAG)
330 return false;
331 break;
332 }
333 default:
334 break;
335 }
336 return true;
337 }
338
cxl_mbox_cmd_ctor(struct cxl_mbox_cmd * mbox,struct cxl_memdev_state * mds,u16 opcode,size_t in_size,size_t out_size,u64 in_payload)339 static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox,
340 struct cxl_memdev_state *mds, u16 opcode,
341 size_t in_size, size_t out_size, u64 in_payload)
342 {
343 *mbox = (struct cxl_mbox_cmd) {
344 .opcode = opcode,
345 .size_in = in_size,
346 };
347
348 if (in_size) {
349 mbox->payload_in = vmemdup_user(u64_to_user_ptr(in_payload),
350 in_size);
351 if (IS_ERR(mbox->payload_in))
352 return PTR_ERR(mbox->payload_in);
353
354 if (!cxl_payload_from_user_allowed(opcode, mbox->payload_in)) {
355 dev_dbg(mds->cxlds.dev, "%s: input payload not allowed\n",
356 cxl_mem_opcode_to_name(opcode));
357 kvfree(mbox->payload_in);
358 return -EBUSY;
359 }
360 }
361
362 /* Prepare to handle a full payload for variable sized output */
363 if (out_size == CXL_VARIABLE_PAYLOAD)
364 mbox->size_out = mds->payload_size;
365 else
366 mbox->size_out = out_size;
367
368 if (mbox->size_out) {
369 mbox->payload_out = kvzalloc(mbox->size_out, GFP_KERNEL);
370 if (!mbox->payload_out) {
371 kvfree(mbox->payload_in);
372 return -ENOMEM;
373 }
374 }
375 return 0;
376 }
377
cxl_mbox_cmd_dtor(struct cxl_mbox_cmd * mbox)378 static void cxl_mbox_cmd_dtor(struct cxl_mbox_cmd *mbox)
379 {
380 kvfree(mbox->payload_in);
381 kvfree(mbox->payload_out);
382 }
383
cxl_to_mem_cmd_raw(struct cxl_mem_command * mem_cmd,const struct cxl_send_command * send_cmd,struct cxl_memdev_state * mds)384 static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd,
385 const struct cxl_send_command *send_cmd,
386 struct cxl_memdev_state *mds)
387 {
388 if (send_cmd->raw.rsvd)
389 return -EINVAL;
390
391 /*
392 * Unlike supported commands, the output size of RAW commands
393 * gets passed along without further checking, so it must be
394 * validated here.
395 */
396 if (send_cmd->out.size > mds->payload_size)
397 return -EINVAL;
398
399 if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode))
400 return -EPERM;
401
402 dev_WARN_ONCE(mds->cxlds.dev, true, "raw command path used\n");
403
404 *mem_cmd = (struct cxl_mem_command) {
405 .info = {
406 .id = CXL_MEM_COMMAND_ID_RAW,
407 .size_in = send_cmd->in.size,
408 .size_out = send_cmd->out.size,
409 },
410 .opcode = send_cmd->raw.opcode
411 };
412
413 return 0;
414 }
415
cxl_to_mem_cmd(struct cxl_mem_command * mem_cmd,const struct cxl_send_command * send_cmd,struct cxl_memdev_state * mds)416 static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd,
417 const struct cxl_send_command *send_cmd,
418 struct cxl_memdev_state *mds)
419 {
420 struct cxl_mem_command *c = &cxl_mem_commands[send_cmd->id];
421 const struct cxl_command_info *info = &c->info;
422
423 if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK)
424 return -EINVAL;
425
426 if (send_cmd->rsvd)
427 return -EINVAL;
428
429 if (send_cmd->in.rsvd || send_cmd->out.rsvd)
430 return -EINVAL;
431
432 /* Check that the command is enabled for hardware */
433 if (!test_bit(info->id, mds->enabled_cmds))
434 return -ENOTTY;
435
436 /* Check that the command is not claimed for exclusive kernel use */
437 if (test_bit(info->id, mds->exclusive_cmds))
438 return -EBUSY;
439
440 /* Check the input buffer is the expected size */
441 if ((info->size_in != CXL_VARIABLE_PAYLOAD) &&
442 (info->size_in != send_cmd->in.size))
443 return -ENOMEM;
444
445 /* Check the output buffer is at least large enough */
446 if ((info->size_out != CXL_VARIABLE_PAYLOAD) &&
447 (send_cmd->out.size < info->size_out))
448 return -ENOMEM;
449
450 *mem_cmd = (struct cxl_mem_command) {
451 .info = {
452 .id = info->id,
453 .flags = info->flags,
454 .size_in = send_cmd->in.size,
455 .size_out = send_cmd->out.size,
456 },
457 .opcode = c->opcode
458 };
459
460 return 0;
461 }
462
463 /**
464 * cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND.
465 * @mbox_cmd: Sanitized and populated &struct cxl_mbox_cmd.
466 * @mds: The driver data for the operation
467 * @send_cmd: &struct cxl_send_command copied in from userspace.
468 *
469 * Return:
470 * * %0 - @out_cmd is ready to send.
471 * * %-ENOTTY - Invalid command specified.
472 * * %-EINVAL - Reserved fields or invalid values were used.
473 * * %-ENOMEM - Input or output buffer wasn't sized properly.
474 * * %-EPERM - Attempted to use a protected command.
475 * * %-EBUSY - Kernel has claimed exclusive access to this opcode
476 *
477 * The result of this command is a fully validated command in @mbox_cmd that is
478 * safe to send to the hardware.
479 */
cxl_validate_cmd_from_user(struct cxl_mbox_cmd * mbox_cmd,struct cxl_memdev_state * mds,const struct cxl_send_command * send_cmd)480 static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,
481 struct cxl_memdev_state *mds,
482 const struct cxl_send_command *send_cmd)
483 {
484 struct cxl_mem_command mem_cmd;
485 int rc;
486
487 if (send_cmd->id == 0 || send_cmd->id >= CXL_MEM_COMMAND_ID_MAX)
488 return -ENOTTY;
489
490 /*
491 * The user can never specify an input payload larger than what hardware
492 * supports, but output can be arbitrarily large (simply write out as
493 * much data as the hardware provides).
494 */
495 if (send_cmd->in.size > mds->payload_size)
496 return -EINVAL;
497
498 /* Sanitize and construct a cxl_mem_command */
499 if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW)
500 rc = cxl_to_mem_cmd_raw(&mem_cmd, send_cmd, mds);
501 else
502 rc = cxl_to_mem_cmd(&mem_cmd, send_cmd, mds);
503
504 if (rc)
505 return rc;
506
507 /* Sanitize and construct a cxl_mbox_cmd */
508 return cxl_mbox_cmd_ctor(mbox_cmd, mds, mem_cmd.opcode,
509 mem_cmd.info.size_in, mem_cmd.info.size_out,
510 send_cmd->in.payload);
511 }
512
cxl_query_cmd(struct cxl_memdev * cxlmd,struct cxl_mem_query_commands __user * q)513 int cxl_query_cmd(struct cxl_memdev *cxlmd,
514 struct cxl_mem_query_commands __user *q)
515 {
516 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
517 struct device *dev = &cxlmd->dev;
518 struct cxl_mem_command *cmd;
519 u32 n_commands;
520 int j = 0;
521
522 dev_dbg(dev, "Query IOCTL\n");
523
524 if (get_user(n_commands, &q->n_commands))
525 return -EFAULT;
526
527 /* returns the total number if 0 elements are requested. */
528 if (n_commands == 0)
529 return put_user(ARRAY_SIZE(cxl_mem_commands), &q->n_commands);
530
531 /*
532 * otherwise, return max(n_commands, total commands) cxl_command_info
533 * structures.
534 */
535 cxl_for_each_cmd(cmd) {
536 struct cxl_command_info info = cmd->info;
537
538 if (test_bit(info.id, mds->enabled_cmds))
539 info.flags |= CXL_MEM_COMMAND_FLAG_ENABLED;
540 if (test_bit(info.id, mds->exclusive_cmds))
541 info.flags |= CXL_MEM_COMMAND_FLAG_EXCLUSIVE;
542
543 if (copy_to_user(&q->commands[j++], &info, sizeof(info)))
544 return -EFAULT;
545
546 if (j == n_commands)
547 break;
548 }
549
550 return 0;
551 }
552
553 /**
554 * handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace.
555 * @mds: The driver data for the operation
556 * @mbox_cmd: The validated mailbox command.
557 * @out_payload: Pointer to userspace's output payload.
558 * @size_out: (Input) Max payload size to copy out.
559 * (Output) Payload size hardware generated.
560 * @retval: Hardware generated return code from the operation.
561 *
562 * Return:
563 * * %0 - Mailbox transaction succeeded. This implies the mailbox
564 * protocol completed successfully not that the operation itself
565 * was successful.
566 * * %-ENOMEM - Couldn't allocate a bounce buffer.
567 * * %-EFAULT - Something happened with copy_to/from_user.
568 * * %-EINTR - Mailbox acquisition interrupted.
569 * * %-EXXX - Transaction level failures.
570 *
571 * Dispatches a mailbox command on behalf of a userspace request.
572 * The output payload is copied to userspace.
573 *
574 * See cxl_send_cmd().
575 */
handle_mailbox_cmd_from_user(struct cxl_memdev_state * mds,struct cxl_mbox_cmd * mbox_cmd,u64 out_payload,s32 * size_out,u32 * retval)576 static int handle_mailbox_cmd_from_user(struct cxl_memdev_state *mds,
577 struct cxl_mbox_cmd *mbox_cmd,
578 u64 out_payload, s32 *size_out,
579 u32 *retval)
580 {
581 struct device *dev = mds->cxlds.dev;
582 int rc;
583
584 dev_dbg(dev,
585 "Submitting %s command for user\n"
586 "\topcode: %x\n"
587 "\tsize: %zx\n",
588 cxl_mem_opcode_to_name(mbox_cmd->opcode),
589 mbox_cmd->opcode, mbox_cmd->size_in);
590
591 rc = mds->mbox_send(mds, mbox_cmd);
592 if (rc)
593 goto out;
594
595 /*
596 * @size_out contains the max size that's allowed to be written back out
597 * to userspace. While the payload may have written more output than
598 * this it will have to be ignored.
599 */
600 if (mbox_cmd->size_out) {
601 dev_WARN_ONCE(dev, mbox_cmd->size_out > *size_out,
602 "Invalid return size\n");
603 if (copy_to_user(u64_to_user_ptr(out_payload),
604 mbox_cmd->payload_out, mbox_cmd->size_out)) {
605 rc = -EFAULT;
606 goto out;
607 }
608 }
609
610 *size_out = mbox_cmd->size_out;
611 *retval = mbox_cmd->return_code;
612
613 out:
614 cxl_mbox_cmd_dtor(mbox_cmd);
615 return rc;
616 }
617
cxl_send_cmd(struct cxl_memdev * cxlmd,struct cxl_send_command __user * s)618 int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s)
619 {
620 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
621 struct device *dev = &cxlmd->dev;
622 struct cxl_send_command send;
623 struct cxl_mbox_cmd mbox_cmd;
624 int rc;
625
626 dev_dbg(dev, "Send IOCTL\n");
627
628 if (copy_from_user(&send, s, sizeof(send)))
629 return -EFAULT;
630
631 rc = cxl_validate_cmd_from_user(&mbox_cmd, mds, &send);
632 if (rc)
633 return rc;
634
635 rc = handle_mailbox_cmd_from_user(mds, &mbox_cmd, send.out.payload,
636 &send.out.size, &send.retval);
637 if (rc)
638 return rc;
639
640 if (copy_to_user(s, &send, sizeof(send)))
641 return -EFAULT;
642
643 return 0;
644 }
645
cxl_xfer_log(struct cxl_memdev_state * mds,uuid_t * uuid,u32 * size,u8 * out)646 static int cxl_xfer_log(struct cxl_memdev_state *mds, uuid_t *uuid,
647 u32 *size, u8 *out)
648 {
649 u32 remaining = *size;
650 u32 offset = 0;
651
652 while (remaining) {
653 u32 xfer_size = min_t(u32, remaining, mds->payload_size);
654 struct cxl_mbox_cmd mbox_cmd;
655 struct cxl_mbox_get_log log;
656 int rc;
657
658 log = (struct cxl_mbox_get_log) {
659 .uuid = *uuid,
660 .offset = cpu_to_le32(offset),
661 .length = cpu_to_le32(xfer_size),
662 };
663
664 mbox_cmd = (struct cxl_mbox_cmd) {
665 .opcode = CXL_MBOX_OP_GET_LOG,
666 .size_in = sizeof(log),
667 .payload_in = &log,
668 .size_out = xfer_size,
669 .payload_out = out,
670 };
671
672 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
673
674 /*
675 * The output payload length that indicates the number
676 * of valid bytes can be smaller than the Log buffer
677 * size.
678 */
679 if (rc == -EIO && mbox_cmd.size_out < xfer_size) {
680 offset += mbox_cmd.size_out;
681 break;
682 }
683
684 if (rc < 0)
685 return rc;
686
687 out += xfer_size;
688 remaining -= xfer_size;
689 offset += xfer_size;
690 }
691
692 *size = offset;
693
694 return 0;
695 }
696
697 /**
698 * cxl_walk_cel() - Walk through the Command Effects Log.
699 * @mds: The driver data for the operation
700 * @size: Length of the Command Effects Log.
701 * @cel: CEL
702 *
703 * Iterate over each entry in the CEL and determine if the driver supports the
704 * command. If so, the command is enabled for the device and can be used later.
705 */
cxl_walk_cel(struct cxl_memdev_state * mds,size_t size,u8 * cel)706 static void cxl_walk_cel(struct cxl_memdev_state *mds, size_t size, u8 *cel)
707 {
708 struct cxl_cel_entry *cel_entry;
709 const int cel_entries = size / sizeof(*cel_entry);
710 struct device *dev = mds->cxlds.dev;
711 int i;
712
713 cel_entry = (struct cxl_cel_entry *) cel;
714
715 for (i = 0; i < cel_entries; i++) {
716 u16 opcode = le16_to_cpu(cel_entry[i].opcode);
717 struct cxl_mem_command *cmd = cxl_mem_find_command(opcode);
718 int enabled = 0;
719
720 if (cmd) {
721 set_bit(cmd->info.id, mds->enabled_cmds);
722 enabled++;
723 }
724
725 if (cxl_is_poison_command(opcode)) {
726 cxl_set_poison_cmd_enabled(&mds->poison, opcode);
727 enabled++;
728 }
729
730 if (cxl_is_security_command(opcode)) {
731 cxl_set_security_cmd_enabled(&mds->security, opcode);
732 enabled++;
733 }
734
735 dev_dbg(dev, "Opcode 0x%04x %s\n", opcode,
736 enabled ? "enabled" : "unsupported by driver");
737 }
738 }
739
cxl_get_gsl(struct cxl_memdev_state * mds)740 static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_memdev_state *mds)
741 {
742 struct cxl_mbox_get_supported_logs *ret;
743 struct cxl_mbox_cmd mbox_cmd;
744 int rc;
745
746 ret = kvmalloc(mds->payload_size, GFP_KERNEL);
747 if (!ret)
748 return ERR_PTR(-ENOMEM);
749
750 mbox_cmd = (struct cxl_mbox_cmd) {
751 .opcode = CXL_MBOX_OP_GET_SUPPORTED_LOGS,
752 .size_out = mds->payload_size,
753 .payload_out = ret,
754 /* At least the record number field must be valid */
755 .min_out = 2,
756 };
757 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
758 if (rc < 0) {
759 kvfree(ret);
760 return ERR_PTR(rc);
761 }
762
763
764 return ret;
765 }
766
767 enum {
768 CEL_UUID,
769 VENDOR_DEBUG_UUID,
770 };
771
772 /* See CXL 2.0 Table 170. Get Log Input Payload */
773 static const uuid_t log_uuid[] = {
774 [CEL_UUID] = DEFINE_CXL_CEL_UUID,
775 [VENDOR_DEBUG_UUID] = DEFINE_CXL_VENDOR_DEBUG_UUID,
776 };
777
778 /**
779 * cxl_enumerate_cmds() - Enumerate commands for a device.
780 * @mds: The driver data for the operation
781 *
782 * Returns 0 if enumerate completed successfully.
783 *
784 * CXL devices have optional support for certain commands. This function will
785 * determine the set of supported commands for the hardware and update the
786 * enabled_cmds bitmap in the @mds.
787 */
cxl_enumerate_cmds(struct cxl_memdev_state * mds)788 int cxl_enumerate_cmds(struct cxl_memdev_state *mds)
789 {
790 struct cxl_mbox_get_supported_logs *gsl;
791 struct device *dev = mds->cxlds.dev;
792 struct cxl_mem_command *cmd;
793 int i, rc;
794
795 gsl = cxl_get_gsl(mds);
796 if (IS_ERR(gsl))
797 return PTR_ERR(gsl);
798
799 rc = -ENOENT;
800 for (i = 0; i < le16_to_cpu(gsl->entries); i++) {
801 u32 size = le32_to_cpu(gsl->entry[i].size);
802 uuid_t uuid = gsl->entry[i].uuid;
803 u8 *log;
804
805 dev_dbg(dev, "Found LOG type %pU of size %d", &uuid, size);
806
807 if (!uuid_equal(&uuid, &log_uuid[CEL_UUID]))
808 continue;
809
810 log = kvmalloc(size, GFP_KERNEL);
811 if (!log) {
812 rc = -ENOMEM;
813 goto out;
814 }
815
816 rc = cxl_xfer_log(mds, &uuid, &size, log);
817 if (rc) {
818 kvfree(log);
819 goto out;
820 }
821
822 cxl_walk_cel(mds, size, log);
823 kvfree(log);
824
825 /* In case CEL was bogus, enable some default commands. */
826 cxl_for_each_cmd(cmd)
827 if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE)
828 set_bit(cmd->info.id, mds->enabled_cmds);
829
830 /* Found the required CEL */
831 rc = 0;
832 }
833 out:
834 kvfree(gsl);
835 return rc;
836 }
837 EXPORT_SYMBOL_NS_GPL(cxl_enumerate_cmds, CXL);
838
839 /*
840 * General Media Event Record
841 * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43
842 */
843 static const uuid_t gen_media_event_uuid =
844 UUID_INIT(0xfbcd0a77, 0xc260, 0x417f,
845 0x85, 0xa9, 0x08, 0x8b, 0x16, 0x21, 0xeb, 0xa6);
846
847 /*
848 * DRAM Event Record
849 * CXL rev 3.0 section 8.2.9.2.1.2; Table 8-44
850 */
851 static const uuid_t dram_event_uuid =
852 UUID_INIT(0x601dcbb3, 0x9c06, 0x4eab,
853 0xb8, 0xaf, 0x4e, 0x9b, 0xfb, 0x5c, 0x96, 0x24);
854
855 /*
856 * Memory Module Event Record
857 * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45
858 */
859 static const uuid_t mem_mod_event_uuid =
860 UUID_INIT(0xfe927475, 0xdd59, 0x4339,
861 0xa5, 0x86, 0x79, 0xba, 0xb1, 0x13, 0xb7, 0x74);
862
cxl_event_trace_record(const struct cxl_memdev * cxlmd,enum cxl_event_log_type type,struct cxl_event_record_raw * record)863 static void cxl_event_trace_record(const struct cxl_memdev *cxlmd,
864 enum cxl_event_log_type type,
865 struct cxl_event_record_raw *record)
866 {
867 uuid_t *id = &record->hdr.id;
868
869 if (uuid_equal(id, &gen_media_event_uuid)) {
870 struct cxl_event_gen_media *rec =
871 (struct cxl_event_gen_media *)record;
872
873 trace_cxl_general_media(cxlmd, type, rec);
874 } else if (uuid_equal(id, &dram_event_uuid)) {
875 struct cxl_event_dram *rec = (struct cxl_event_dram *)record;
876
877 trace_cxl_dram(cxlmd, type, rec);
878 } else if (uuid_equal(id, &mem_mod_event_uuid)) {
879 struct cxl_event_mem_module *rec =
880 (struct cxl_event_mem_module *)record;
881
882 trace_cxl_memory_module(cxlmd, type, rec);
883 } else {
884 /* For unknown record types print just the header */
885 trace_cxl_generic_event(cxlmd, type, record);
886 }
887 }
888
cxl_clear_event_record(struct cxl_memdev_state * mds,enum cxl_event_log_type log,struct cxl_get_event_payload * get_pl)889 static int cxl_clear_event_record(struct cxl_memdev_state *mds,
890 enum cxl_event_log_type log,
891 struct cxl_get_event_payload *get_pl)
892 {
893 struct cxl_mbox_clear_event_payload *payload;
894 u16 total = le16_to_cpu(get_pl->record_count);
895 u8 max_handles = CXL_CLEAR_EVENT_MAX_HANDLES;
896 size_t pl_size = struct_size(payload, handles, max_handles);
897 struct cxl_mbox_cmd mbox_cmd;
898 u16 cnt;
899 int rc = 0;
900 int i;
901
902 /* Payload size may limit the max handles */
903 if (pl_size > mds->payload_size) {
904 max_handles = (mds->payload_size - sizeof(*payload)) /
905 sizeof(__le16);
906 pl_size = struct_size(payload, handles, max_handles);
907 }
908
909 payload = kvzalloc(pl_size, GFP_KERNEL);
910 if (!payload)
911 return -ENOMEM;
912
913 *payload = (struct cxl_mbox_clear_event_payload) {
914 .event_log = log,
915 };
916
917 mbox_cmd = (struct cxl_mbox_cmd) {
918 .opcode = CXL_MBOX_OP_CLEAR_EVENT_RECORD,
919 .payload_in = payload,
920 .size_in = pl_size,
921 };
922
923 /*
924 * Clear Event Records uses u8 for the handle cnt while Get Event
925 * Record can return up to 0xffff records.
926 */
927 i = 0;
928 for (cnt = 0; cnt < total; cnt++) {
929 payload->handles[i++] = get_pl->records[cnt].hdr.handle;
930 dev_dbg(mds->cxlds.dev, "Event log '%d': Clearing %u\n", log,
931 le16_to_cpu(payload->handles[i - 1]));
932
933 if (i == max_handles) {
934 payload->nr_recs = i;
935 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
936 if (rc)
937 goto free_pl;
938 i = 0;
939 }
940 }
941
942 /* Clear what is left if any */
943 if (i) {
944 payload->nr_recs = i;
945 mbox_cmd.size_in = struct_size(payload, handles, i);
946 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
947 if (rc)
948 goto free_pl;
949 }
950
951 free_pl:
952 kvfree(payload);
953 return rc;
954 }
955
cxl_mem_get_records_log(struct cxl_memdev_state * mds,enum cxl_event_log_type type)956 static void cxl_mem_get_records_log(struct cxl_memdev_state *mds,
957 enum cxl_event_log_type type)
958 {
959 struct cxl_memdev *cxlmd = mds->cxlds.cxlmd;
960 struct device *dev = mds->cxlds.dev;
961 struct cxl_get_event_payload *payload;
962 u8 log_type = type;
963 u16 nr_rec;
964
965 mutex_lock(&mds->event.log_lock);
966 payload = mds->event.buf;
967
968 do {
969 int rc, i;
970 struct cxl_mbox_cmd mbox_cmd = (struct cxl_mbox_cmd) {
971 .opcode = CXL_MBOX_OP_GET_EVENT_RECORD,
972 .payload_in = &log_type,
973 .size_in = sizeof(log_type),
974 .payload_out = payload,
975 .size_out = mds->payload_size,
976 .min_out = struct_size(payload, records, 0),
977 };
978
979 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
980 if (rc) {
981 dev_err_ratelimited(dev,
982 "Event log '%d': Failed to query event records : %d",
983 type, rc);
984 break;
985 }
986
987 nr_rec = le16_to_cpu(payload->record_count);
988 if (!nr_rec)
989 break;
990
991 for (i = 0; i < nr_rec; i++)
992 cxl_event_trace_record(cxlmd, type,
993 &payload->records[i]);
994
995 if (payload->flags & CXL_GET_EVENT_FLAG_OVERFLOW)
996 trace_cxl_overflow(cxlmd, type, payload);
997
998 rc = cxl_clear_event_record(mds, type, payload);
999 if (rc) {
1000 dev_err_ratelimited(dev,
1001 "Event log '%d': Failed to clear events : %d",
1002 type, rc);
1003 break;
1004 }
1005 } while (nr_rec);
1006
1007 mutex_unlock(&mds->event.log_lock);
1008 }
1009
1010 /**
1011 * cxl_mem_get_event_records - Get Event Records from the device
1012 * @mds: The driver data for the operation
1013 * @status: Event Status register value identifying which events are available.
1014 *
1015 * Retrieve all event records available on the device, report them as trace
1016 * events, and clear them.
1017 *
1018 * See CXL rev 3.0 @8.2.9.2.2 Get Event Records
1019 * See CXL rev 3.0 @8.2.9.2.3 Clear Event Records
1020 */
cxl_mem_get_event_records(struct cxl_memdev_state * mds,u32 status)1021 void cxl_mem_get_event_records(struct cxl_memdev_state *mds, u32 status)
1022 {
1023 dev_dbg(mds->cxlds.dev, "Reading event logs: %x\n", status);
1024
1025 if (status & CXLDEV_EVENT_STATUS_FATAL)
1026 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_FATAL);
1027 if (status & CXLDEV_EVENT_STATUS_FAIL)
1028 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_FAIL);
1029 if (status & CXLDEV_EVENT_STATUS_WARN)
1030 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_WARN);
1031 if (status & CXLDEV_EVENT_STATUS_INFO)
1032 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_INFO);
1033 }
1034 EXPORT_SYMBOL_NS_GPL(cxl_mem_get_event_records, CXL);
1035
1036 /**
1037 * cxl_mem_get_partition_info - Get partition info
1038 * @mds: The driver data for the operation
1039 *
1040 * Retrieve the current partition info for the device specified. The active
1041 * values are the current capacity in bytes. If not 0, the 'next' values are
1042 * the pending values, in bytes, which take affect on next cold reset.
1043 *
1044 * Return: 0 if no error: or the result of the mailbox command.
1045 *
1046 * See CXL @8.2.9.5.2.1 Get Partition Info
1047 */
cxl_mem_get_partition_info(struct cxl_memdev_state * mds)1048 static int cxl_mem_get_partition_info(struct cxl_memdev_state *mds)
1049 {
1050 struct cxl_mbox_get_partition_info pi;
1051 struct cxl_mbox_cmd mbox_cmd;
1052 int rc;
1053
1054 mbox_cmd = (struct cxl_mbox_cmd) {
1055 .opcode = CXL_MBOX_OP_GET_PARTITION_INFO,
1056 .size_out = sizeof(pi),
1057 .payload_out = &pi,
1058 };
1059 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
1060 if (rc)
1061 return rc;
1062
1063 mds->active_volatile_bytes =
1064 le64_to_cpu(pi.active_volatile_cap) * CXL_CAPACITY_MULTIPLIER;
1065 mds->active_persistent_bytes =
1066 le64_to_cpu(pi.active_persistent_cap) * CXL_CAPACITY_MULTIPLIER;
1067 mds->next_volatile_bytes =
1068 le64_to_cpu(pi.next_volatile_cap) * CXL_CAPACITY_MULTIPLIER;
1069 mds->next_persistent_bytes =
1070 le64_to_cpu(pi.next_volatile_cap) * CXL_CAPACITY_MULTIPLIER;
1071
1072 return 0;
1073 }
1074
1075 /**
1076 * cxl_dev_state_identify() - Send the IDENTIFY command to the device.
1077 * @mds: The driver data for the operation
1078 *
1079 * Return: 0 if identify was executed successfully or media not ready.
1080 *
1081 * This will dispatch the identify command to the device and on success populate
1082 * structures to be exported to sysfs.
1083 */
cxl_dev_state_identify(struct cxl_memdev_state * mds)1084 int cxl_dev_state_identify(struct cxl_memdev_state *mds)
1085 {
1086 /* See CXL 2.0 Table 175 Identify Memory Device Output Payload */
1087 struct cxl_mbox_identify id;
1088 struct cxl_mbox_cmd mbox_cmd;
1089 u32 val;
1090 int rc;
1091
1092 if (!mds->cxlds.media_ready)
1093 return 0;
1094
1095 mbox_cmd = (struct cxl_mbox_cmd) {
1096 .opcode = CXL_MBOX_OP_IDENTIFY,
1097 .size_out = sizeof(id),
1098 .payload_out = &id,
1099 };
1100 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
1101 if (rc < 0)
1102 return rc;
1103
1104 mds->total_bytes =
1105 le64_to_cpu(id.total_capacity) * CXL_CAPACITY_MULTIPLIER;
1106 mds->volatile_only_bytes =
1107 le64_to_cpu(id.volatile_capacity) * CXL_CAPACITY_MULTIPLIER;
1108 mds->persistent_only_bytes =
1109 le64_to_cpu(id.persistent_capacity) * CXL_CAPACITY_MULTIPLIER;
1110 mds->partition_align_bytes =
1111 le64_to_cpu(id.partition_align) * CXL_CAPACITY_MULTIPLIER;
1112
1113 mds->lsa_size = le32_to_cpu(id.lsa_size);
1114 memcpy(mds->firmware_version, id.fw_revision,
1115 sizeof(id.fw_revision));
1116
1117 if (test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds)) {
1118 val = get_unaligned_le24(id.poison_list_max_mer);
1119 mds->poison.max_errors = min_t(u32, val, CXL_POISON_LIST_MAX);
1120 }
1121
1122 return 0;
1123 }
1124 EXPORT_SYMBOL_NS_GPL(cxl_dev_state_identify, CXL);
1125
__cxl_mem_sanitize(struct cxl_memdev_state * mds,u16 cmd)1126 static int __cxl_mem_sanitize(struct cxl_memdev_state *mds, u16 cmd)
1127 {
1128 int rc;
1129 u32 sec_out = 0;
1130 struct cxl_get_security_output {
1131 __le32 flags;
1132 } out;
1133 struct cxl_mbox_cmd sec_cmd = {
1134 .opcode = CXL_MBOX_OP_GET_SECURITY_STATE,
1135 .payload_out = &out,
1136 .size_out = sizeof(out),
1137 };
1138 struct cxl_mbox_cmd mbox_cmd = { .opcode = cmd };
1139 struct cxl_dev_state *cxlds = &mds->cxlds;
1140
1141 if (cmd != CXL_MBOX_OP_SANITIZE && cmd != CXL_MBOX_OP_SECURE_ERASE)
1142 return -EINVAL;
1143
1144 rc = cxl_internal_send_cmd(mds, &sec_cmd);
1145 if (rc < 0) {
1146 dev_err(cxlds->dev, "Failed to get security state : %d", rc);
1147 return rc;
1148 }
1149
1150 /*
1151 * Prior to using these commands, any security applied to
1152 * the user data areas of the device shall be DISABLED (or
1153 * UNLOCKED for secure erase case).
1154 */
1155 sec_out = le32_to_cpu(out.flags);
1156 if (sec_out & CXL_PMEM_SEC_STATE_USER_PASS_SET)
1157 return -EINVAL;
1158
1159 if (cmd == CXL_MBOX_OP_SECURE_ERASE &&
1160 sec_out & CXL_PMEM_SEC_STATE_LOCKED)
1161 return -EINVAL;
1162
1163 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
1164 if (rc < 0) {
1165 dev_err(cxlds->dev, "Failed to sanitize device : %d", rc);
1166 return rc;
1167 }
1168
1169 return 0;
1170 }
1171
1172
1173 /**
1174 * cxl_mem_sanitize() - Send a sanitization command to the device.
1175 * @cxlmd: The device for the operation
1176 * @cmd: The specific sanitization command opcode
1177 *
1178 * Return: 0 if the command was executed successfully, regardless of
1179 * whether or not the actual security operation is done in the background,
1180 * such as for the Sanitize case.
1181 * Error return values can be the result of the mailbox command, -EINVAL
1182 * when security requirements are not met or invalid contexts, or -EBUSY
1183 * if the sanitize operation is already in flight.
1184 *
1185 * See CXL 3.0 @8.2.9.8.5.1 Sanitize and @8.2.9.8.5.2 Secure Erase.
1186 */
cxl_mem_sanitize(struct cxl_memdev * cxlmd,u16 cmd)1187 int cxl_mem_sanitize(struct cxl_memdev *cxlmd, u16 cmd)
1188 {
1189 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
1190 struct cxl_port *endpoint;
1191 int rc;
1192
1193 /* synchronize with cxl_mem_probe() and decoder write operations */
1194 device_lock(&cxlmd->dev);
1195 endpoint = cxlmd->endpoint;
1196 down_read(&cxl_region_rwsem);
1197 /*
1198 * Require an endpoint to be safe otherwise the driver can not
1199 * be sure that the device is unmapped.
1200 */
1201 if (endpoint && cxl_num_decoders_committed(endpoint) == 0)
1202 rc = __cxl_mem_sanitize(mds, cmd);
1203 else
1204 rc = -EBUSY;
1205 up_read(&cxl_region_rwsem);
1206 device_unlock(&cxlmd->dev);
1207
1208 return rc;
1209 }
1210
add_dpa_res(struct device * dev,struct resource * parent,struct resource * res,resource_size_t start,resource_size_t size,const char * type)1211 static int add_dpa_res(struct device *dev, struct resource *parent,
1212 struct resource *res, resource_size_t start,
1213 resource_size_t size, const char *type)
1214 {
1215 int rc;
1216
1217 res->name = type;
1218 res->start = start;
1219 res->end = start + size - 1;
1220 res->flags = IORESOURCE_MEM;
1221 if (resource_size(res) == 0) {
1222 dev_dbg(dev, "DPA(%s): no capacity\n", res->name);
1223 return 0;
1224 }
1225 rc = request_resource(parent, res);
1226 if (rc) {
1227 dev_err(dev, "DPA(%s): failed to track %pr (%d)\n", res->name,
1228 res, rc);
1229 return rc;
1230 }
1231
1232 dev_dbg(dev, "DPA(%s): %pr\n", res->name, res);
1233
1234 return 0;
1235 }
1236
cxl_mem_create_range_info(struct cxl_memdev_state * mds)1237 int cxl_mem_create_range_info(struct cxl_memdev_state *mds)
1238 {
1239 struct cxl_dev_state *cxlds = &mds->cxlds;
1240 struct device *dev = cxlds->dev;
1241 int rc;
1242
1243 if (!cxlds->media_ready) {
1244 cxlds->dpa_res = DEFINE_RES_MEM(0, 0);
1245 cxlds->ram_res = DEFINE_RES_MEM(0, 0);
1246 cxlds->pmem_res = DEFINE_RES_MEM(0, 0);
1247 return 0;
1248 }
1249
1250 cxlds->dpa_res =
1251 (struct resource)DEFINE_RES_MEM(0, mds->total_bytes);
1252
1253 if (mds->partition_align_bytes == 0) {
1254 rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0,
1255 mds->volatile_only_bytes, "ram");
1256 if (rc)
1257 return rc;
1258 return add_dpa_res(dev, &cxlds->dpa_res, &cxlds->pmem_res,
1259 mds->volatile_only_bytes,
1260 mds->persistent_only_bytes, "pmem");
1261 }
1262
1263 rc = cxl_mem_get_partition_info(mds);
1264 if (rc) {
1265 dev_err(dev, "Failed to query partition information\n");
1266 return rc;
1267 }
1268
1269 rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0,
1270 mds->active_volatile_bytes, "ram");
1271 if (rc)
1272 return rc;
1273 return add_dpa_res(dev, &cxlds->dpa_res, &cxlds->pmem_res,
1274 mds->active_volatile_bytes,
1275 mds->active_persistent_bytes, "pmem");
1276 }
1277 EXPORT_SYMBOL_NS_GPL(cxl_mem_create_range_info, CXL);
1278
cxl_set_timestamp(struct cxl_memdev_state * mds)1279 int cxl_set_timestamp(struct cxl_memdev_state *mds)
1280 {
1281 struct cxl_mbox_cmd mbox_cmd;
1282 struct cxl_mbox_set_timestamp_in pi;
1283 int rc;
1284
1285 pi.timestamp = cpu_to_le64(ktime_get_real_ns());
1286 mbox_cmd = (struct cxl_mbox_cmd) {
1287 .opcode = CXL_MBOX_OP_SET_TIMESTAMP,
1288 .size_in = sizeof(pi),
1289 .payload_in = &pi,
1290 };
1291
1292 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
1293 /*
1294 * Command is optional. Devices may have another way of providing
1295 * a timestamp, or may return all 0s in timestamp fields.
1296 * Don't report an error if this command isn't supported
1297 */
1298 if (rc && (mbox_cmd.return_code != CXL_MBOX_CMD_RC_UNSUPPORTED))
1299 return rc;
1300
1301 return 0;
1302 }
1303 EXPORT_SYMBOL_NS_GPL(cxl_set_timestamp, CXL);
1304
cxl_mem_get_poison(struct cxl_memdev * cxlmd,u64 offset,u64 len,struct cxl_region * cxlr)1305 int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
1306 struct cxl_region *cxlr)
1307 {
1308 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
1309 struct cxl_mbox_poison_out *po;
1310 struct cxl_mbox_poison_in pi;
1311 int nr_records = 0;
1312 int rc;
1313
1314 rc = mutex_lock_interruptible(&mds->poison.lock);
1315 if (rc)
1316 return rc;
1317
1318 po = mds->poison.list_out;
1319 pi.offset = cpu_to_le64(offset);
1320 pi.length = cpu_to_le64(len / CXL_POISON_LEN_MULT);
1321
1322 do {
1323 struct cxl_mbox_cmd mbox_cmd = (struct cxl_mbox_cmd){
1324 .opcode = CXL_MBOX_OP_GET_POISON,
1325 .size_in = sizeof(pi),
1326 .payload_in = &pi,
1327 .size_out = mds->payload_size,
1328 .payload_out = po,
1329 .min_out = struct_size(po, record, 0),
1330 };
1331
1332 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
1333 if (rc)
1334 break;
1335
1336 for (int i = 0; i < le16_to_cpu(po->count); i++)
1337 trace_cxl_poison(cxlmd, cxlr, &po->record[i],
1338 po->flags, po->overflow_ts,
1339 CXL_POISON_TRACE_LIST);
1340
1341 /* Protect against an uncleared _FLAG_MORE */
1342 nr_records = nr_records + le16_to_cpu(po->count);
1343 if (nr_records >= mds->poison.max_errors) {
1344 dev_dbg(&cxlmd->dev, "Max Error Records reached: %d\n",
1345 nr_records);
1346 break;
1347 }
1348 } while (po->flags & CXL_POISON_FLAG_MORE);
1349
1350 mutex_unlock(&mds->poison.lock);
1351 return rc;
1352 }
1353 EXPORT_SYMBOL_NS_GPL(cxl_mem_get_poison, CXL);
1354
free_poison_buf(void * buf)1355 static void free_poison_buf(void *buf)
1356 {
1357 kvfree(buf);
1358 }
1359
1360 /* Get Poison List output buffer is protected by mds->poison.lock */
cxl_poison_alloc_buf(struct cxl_memdev_state * mds)1361 static int cxl_poison_alloc_buf(struct cxl_memdev_state *mds)
1362 {
1363 mds->poison.list_out = kvmalloc(mds->payload_size, GFP_KERNEL);
1364 if (!mds->poison.list_out)
1365 return -ENOMEM;
1366
1367 return devm_add_action_or_reset(mds->cxlds.dev, free_poison_buf,
1368 mds->poison.list_out);
1369 }
1370
cxl_poison_state_init(struct cxl_memdev_state * mds)1371 int cxl_poison_state_init(struct cxl_memdev_state *mds)
1372 {
1373 int rc;
1374
1375 if (!test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds))
1376 return 0;
1377
1378 rc = cxl_poison_alloc_buf(mds);
1379 if (rc) {
1380 clear_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds);
1381 return rc;
1382 }
1383
1384 mutex_init(&mds->poison.lock);
1385 return 0;
1386 }
1387 EXPORT_SYMBOL_NS_GPL(cxl_poison_state_init, CXL);
1388
cxl_memdev_state_create(struct device * dev)1389 struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev)
1390 {
1391 struct cxl_memdev_state *mds;
1392
1393 mds = devm_kzalloc(dev, sizeof(*mds), GFP_KERNEL);
1394 if (!mds) {
1395 dev_err(dev, "No memory available\n");
1396 return ERR_PTR(-ENOMEM);
1397 }
1398
1399 mutex_init(&mds->mbox_mutex);
1400 mutex_init(&mds->event.log_lock);
1401 mds->cxlds.dev = dev;
1402 mds->cxlds.type = CXL_DEVTYPE_CLASSMEM;
1403
1404 return mds;
1405 }
1406 EXPORT_SYMBOL_NS_GPL(cxl_memdev_state_create, CXL);
1407
cxl_mbox_init(void)1408 void __init cxl_mbox_init(void)
1409 {
1410 struct dentry *mbox_debugfs;
1411
1412 mbox_debugfs = cxl_debugfs_create_dir("mbox");
1413 debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs,
1414 &cxl_raw_allow_all);
1415 }
1416