1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2020 Intel Corporation. */ 3 4 #include <linux/io-64-nonatomic-lo-hi.h> 5 #include <linux/firmware.h> 6 #include <linux/device.h> 7 #include <linux/slab.h> 8 #include <linux/idr.h> 9 #include <linux/pci.h> 10 #include <cxlmem.h> 11 #include "trace.h" 12 #include "core.h" 13 14 static DECLARE_RWSEM(cxl_memdev_rwsem); 15 16 /* 17 * An entire PCI topology full of devices should be enough for any 18 * config 19 */ 20 #define CXL_MEM_MAX_DEVS 65536 21 22 static int cxl_mem_major; 23 static DEFINE_IDA(cxl_memdev_ida); 24 25 static void cxl_memdev_release(struct device *dev) 26 { 27 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 28 29 ida_free(&cxl_memdev_ida, cxlmd->id); 30 kfree(cxlmd); 31 } 32 33 static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid, 34 kgid_t *gid) 35 { 36 return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); 37 } 38 39 static ssize_t firmware_version_show(struct device *dev, 40 struct device_attribute *attr, char *buf) 41 { 42 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 43 struct cxl_dev_state *cxlds = cxlmd->cxlds; 44 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 45 46 if (!mds) 47 return sysfs_emit(buf, "\n"); 48 return sysfs_emit(buf, "%.16s\n", mds->firmware_version); 49 } 50 static DEVICE_ATTR_RO(firmware_version); 51 52 static ssize_t payload_max_show(struct device *dev, 53 struct device_attribute *attr, char *buf) 54 { 55 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 56 struct cxl_dev_state *cxlds = cxlmd->cxlds; 57 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 58 59 if (!mds) 60 return sysfs_emit(buf, "\n"); 61 return sysfs_emit(buf, "%zu\n", mds->payload_size); 62 } 63 static DEVICE_ATTR_RO(payload_max); 64 65 static ssize_t label_storage_size_show(struct device *dev, 66 struct device_attribute *attr, char *buf) 67 { 68 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 69 struct cxl_dev_state *cxlds = cxlmd->cxlds; 70 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 71 72 if (!mds) 73 return sysfs_emit(buf, "\n"); 74 return sysfs_emit(buf, "%zu\n", mds->lsa_size); 75 } 76 static DEVICE_ATTR_RO(label_storage_size); 77 78 static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, 79 char *buf) 80 { 81 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 82 struct cxl_dev_state *cxlds = cxlmd->cxlds; 83 unsigned long long len = resource_size(&cxlds->ram_res); 84 85 return sysfs_emit(buf, "%#llx\n", len); 86 } 87 88 static struct device_attribute dev_attr_ram_size = 89 __ATTR(size, 0444, ram_size_show, NULL); 90 91 static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, 92 char *buf) 93 { 94 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 95 struct cxl_dev_state *cxlds = cxlmd->cxlds; 96 unsigned long long len = resource_size(&cxlds->pmem_res); 97 98 return sysfs_emit(buf, "%#llx\n", len); 99 } 100 101 static struct device_attribute dev_attr_pmem_size = 102 __ATTR(size, 0444, pmem_size_show, NULL); 103 104 static ssize_t serial_show(struct device *dev, struct device_attribute *attr, 105 char *buf) 106 { 107 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 108 struct cxl_dev_state *cxlds = cxlmd->cxlds; 109 110 return sysfs_emit(buf, "%#llx\n", cxlds->serial); 111 } 112 static DEVICE_ATTR_RO(serial); 113 114 static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, 115 char *buf) 116 { 117 return sprintf(buf, "%d\n", dev_to_node(dev)); 118 } 119 static DEVICE_ATTR_RO(numa_node); 120 121 static ssize_t security_state_show(struct device *dev, 122 struct device_attribute *attr, 123 char *buf) 124 { 125 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 126 struct cxl_dev_state *cxlds = cxlmd->cxlds; 127 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 128 u64 reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_BG_CMD_STATUS_OFFSET); 129 u32 pct = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_PCT_MASK, reg); 130 u16 cmd = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_OPCODE_MASK, reg); 131 unsigned long state = mds->security.state; 132 133 if (cmd == CXL_MBOX_OP_SANITIZE && pct != 100) 134 return sysfs_emit(buf, "sanitize\n"); 135 136 if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET)) 137 return sysfs_emit(buf, "disabled\n"); 138 if (state & CXL_PMEM_SEC_STATE_FROZEN || 139 state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT || 140 state & CXL_PMEM_SEC_STATE_USER_PLIMIT) 141 return sysfs_emit(buf, "frozen\n"); 142 if (state & CXL_PMEM_SEC_STATE_LOCKED) 143 return sysfs_emit(buf, "locked\n"); 144 else 145 return sysfs_emit(buf, "unlocked\n"); 146 } 147 static struct device_attribute dev_attr_security_state = 148 __ATTR(state, 0444, security_state_show, NULL); 149 150 static ssize_t security_sanitize_store(struct device *dev, 151 struct device_attribute *attr, 152 const char *buf, size_t len) 153 { 154 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 155 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 156 struct cxl_port *port = cxlmd->endpoint; 157 bool sanitize; 158 ssize_t rc; 159 160 if (kstrtobool(buf, &sanitize) || !sanitize) 161 return -EINVAL; 162 163 if (!port || !is_cxl_endpoint(port)) 164 return -EINVAL; 165 166 /* ensure no regions are mapped to this memdev */ 167 if (port->commit_end != -1) 168 return -EBUSY; 169 170 rc = cxl_mem_sanitize(mds, CXL_MBOX_OP_SANITIZE); 171 172 return rc ? rc : len; 173 } 174 static struct device_attribute dev_attr_security_sanitize = 175 __ATTR(sanitize, 0200, NULL, security_sanitize_store); 176 177 static ssize_t security_erase_store(struct device *dev, 178 struct device_attribute *attr, 179 const char *buf, size_t len) 180 { 181 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 182 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 183 struct cxl_port *port = cxlmd->endpoint; 184 ssize_t rc; 185 bool erase; 186 187 if (kstrtobool(buf, &erase) || !erase) 188 return -EINVAL; 189 190 if (!port || !is_cxl_endpoint(port)) 191 return -EINVAL; 192 193 /* ensure no regions are mapped to this memdev */ 194 if (port->commit_end != -1) 195 return -EBUSY; 196 197 rc = cxl_mem_sanitize(mds, CXL_MBOX_OP_SECURE_ERASE); 198 199 return rc ? rc : len; 200 } 201 static struct device_attribute dev_attr_security_erase = 202 __ATTR(erase, 0200, NULL, security_erase_store); 203 204 static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd) 205 { 206 struct cxl_dev_state *cxlds = cxlmd->cxlds; 207 u64 offset, length; 208 int rc = 0; 209 210 /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */ 211 if (resource_size(&cxlds->pmem_res)) { 212 offset = cxlds->pmem_res.start; 213 length = resource_size(&cxlds->pmem_res); 214 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 215 if (rc) 216 return rc; 217 } 218 if (resource_size(&cxlds->ram_res)) { 219 offset = cxlds->ram_res.start; 220 length = resource_size(&cxlds->ram_res); 221 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 222 /* 223 * Invalid Physical Address is not an error for 224 * volatile addresses. Device support is optional. 225 */ 226 if (rc == -EFAULT) 227 rc = 0; 228 } 229 return rc; 230 } 231 232 int cxl_trigger_poison_list(struct cxl_memdev *cxlmd) 233 { 234 struct cxl_port *port; 235 int rc; 236 237 port = cxlmd->endpoint; 238 if (!port || !is_cxl_endpoint(port)) 239 return -EINVAL; 240 241 rc = down_read_interruptible(&cxl_dpa_rwsem); 242 if (rc) 243 return rc; 244 245 if (port->commit_end == -1) { 246 /* No regions mapped to this memdev */ 247 rc = cxl_get_poison_by_memdev(cxlmd); 248 } else { 249 /* Regions mapped, collect poison by endpoint */ 250 rc = cxl_get_poison_by_endpoint(port); 251 } 252 up_read(&cxl_dpa_rwsem); 253 254 return rc; 255 } 256 EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, CXL); 257 258 struct cxl_dpa_to_region_context { 259 struct cxl_region *cxlr; 260 u64 dpa; 261 }; 262 263 static int __cxl_dpa_to_region(struct device *dev, void *arg) 264 { 265 struct cxl_dpa_to_region_context *ctx = arg; 266 struct cxl_endpoint_decoder *cxled; 267 u64 dpa = ctx->dpa; 268 269 if (!is_endpoint_decoder(dev)) 270 return 0; 271 272 cxled = to_cxl_endpoint_decoder(dev); 273 if (!cxled->dpa_res || !resource_size(cxled->dpa_res)) 274 return 0; 275 276 if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start) 277 return 0; 278 279 dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa, 280 dev_name(&cxled->cxld.region->dev)); 281 282 ctx->cxlr = cxled->cxld.region; 283 284 return 1; 285 } 286 287 static struct cxl_region *cxl_dpa_to_region(struct cxl_memdev *cxlmd, u64 dpa) 288 { 289 struct cxl_dpa_to_region_context ctx; 290 struct cxl_port *port; 291 292 ctx = (struct cxl_dpa_to_region_context) { 293 .dpa = dpa, 294 }; 295 port = cxlmd->endpoint; 296 if (port && is_cxl_endpoint(port) && port->commit_end != -1) 297 device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region); 298 299 return ctx.cxlr; 300 } 301 302 static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa) 303 { 304 struct cxl_dev_state *cxlds = cxlmd->cxlds; 305 306 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 307 return 0; 308 309 if (!resource_size(&cxlds->dpa_res)) { 310 dev_dbg(cxlds->dev, "device has no dpa resource\n"); 311 return -EINVAL; 312 } 313 if (dpa < cxlds->dpa_res.start || dpa > cxlds->dpa_res.end) { 314 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n", 315 dpa, &cxlds->dpa_res); 316 return -EINVAL; 317 } 318 if (!IS_ALIGNED(dpa, 64)) { 319 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa); 320 return -EINVAL; 321 } 322 323 return 0; 324 } 325 326 int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa) 327 { 328 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 329 struct cxl_mbox_inject_poison inject; 330 struct cxl_poison_record record; 331 struct cxl_mbox_cmd mbox_cmd; 332 struct cxl_region *cxlr; 333 int rc; 334 335 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 336 return 0; 337 338 rc = down_read_interruptible(&cxl_dpa_rwsem); 339 if (rc) 340 return rc; 341 342 rc = cxl_validate_poison_dpa(cxlmd, dpa); 343 if (rc) 344 goto out; 345 346 inject.address = cpu_to_le64(dpa); 347 mbox_cmd = (struct cxl_mbox_cmd) { 348 .opcode = CXL_MBOX_OP_INJECT_POISON, 349 .size_in = sizeof(inject), 350 .payload_in = &inject, 351 }; 352 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 353 if (rc) 354 goto out; 355 356 cxlr = cxl_dpa_to_region(cxlmd, dpa); 357 if (cxlr) 358 dev_warn_once(mds->cxlds.dev, 359 "poison inject dpa:%#llx region: %s\n", dpa, 360 dev_name(&cxlr->dev)); 361 362 record = (struct cxl_poison_record) { 363 .address = cpu_to_le64(dpa), 364 .length = cpu_to_le32(1), 365 }; 366 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT); 367 out: 368 up_read(&cxl_dpa_rwsem); 369 370 return rc; 371 } 372 EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, CXL); 373 374 int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa) 375 { 376 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 377 struct cxl_mbox_clear_poison clear; 378 struct cxl_poison_record record; 379 struct cxl_mbox_cmd mbox_cmd; 380 struct cxl_region *cxlr; 381 int rc; 382 383 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 384 return 0; 385 386 rc = down_read_interruptible(&cxl_dpa_rwsem); 387 if (rc) 388 return rc; 389 390 rc = cxl_validate_poison_dpa(cxlmd, dpa); 391 if (rc) 392 goto out; 393 394 /* 395 * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command 396 * is defined to accept 64 bytes of write-data, along with the 397 * address to clear. This driver uses zeroes as write-data. 398 */ 399 clear = (struct cxl_mbox_clear_poison) { 400 .address = cpu_to_le64(dpa) 401 }; 402 403 mbox_cmd = (struct cxl_mbox_cmd) { 404 .opcode = CXL_MBOX_OP_CLEAR_POISON, 405 .size_in = sizeof(clear), 406 .payload_in = &clear, 407 }; 408 409 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 410 if (rc) 411 goto out; 412 413 cxlr = cxl_dpa_to_region(cxlmd, dpa); 414 if (cxlr) 415 dev_warn_once(mds->cxlds.dev, 416 "poison clear dpa:%#llx region: %s\n", dpa, 417 dev_name(&cxlr->dev)); 418 419 record = (struct cxl_poison_record) { 420 .address = cpu_to_le64(dpa), 421 .length = cpu_to_le32(1), 422 }; 423 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR); 424 out: 425 up_read(&cxl_dpa_rwsem); 426 427 return rc; 428 } 429 EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, CXL); 430 431 static struct attribute *cxl_memdev_attributes[] = { 432 &dev_attr_serial.attr, 433 &dev_attr_firmware_version.attr, 434 &dev_attr_payload_max.attr, 435 &dev_attr_label_storage_size.attr, 436 &dev_attr_numa_node.attr, 437 NULL, 438 }; 439 440 static struct attribute *cxl_memdev_pmem_attributes[] = { 441 &dev_attr_pmem_size.attr, 442 NULL, 443 }; 444 445 static struct attribute *cxl_memdev_ram_attributes[] = { 446 &dev_attr_ram_size.attr, 447 NULL, 448 }; 449 450 static struct attribute *cxl_memdev_security_attributes[] = { 451 &dev_attr_security_state.attr, 452 &dev_attr_security_sanitize.attr, 453 &dev_attr_security_erase.attr, 454 NULL, 455 }; 456 457 static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a, 458 int n) 459 { 460 if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr) 461 return 0; 462 return a->mode; 463 } 464 465 static struct attribute_group cxl_memdev_attribute_group = { 466 .attrs = cxl_memdev_attributes, 467 .is_visible = cxl_memdev_visible, 468 }; 469 470 static struct attribute_group cxl_memdev_ram_attribute_group = { 471 .name = "ram", 472 .attrs = cxl_memdev_ram_attributes, 473 }; 474 475 static struct attribute_group cxl_memdev_pmem_attribute_group = { 476 .name = "pmem", 477 .attrs = cxl_memdev_pmem_attributes, 478 }; 479 480 static umode_t cxl_memdev_security_visible(struct kobject *kobj, 481 struct attribute *a, int n) 482 { 483 struct device *dev = kobj_to_dev(kobj); 484 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 485 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 486 487 if (a == &dev_attr_security_sanitize.attr && 488 !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) 489 return 0; 490 491 if (a == &dev_attr_security_erase.attr && 492 !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds)) 493 return 0; 494 495 return a->mode; 496 } 497 498 static struct attribute_group cxl_memdev_security_attribute_group = { 499 .name = "security", 500 .attrs = cxl_memdev_security_attributes, 501 .is_visible = cxl_memdev_security_visible, 502 }; 503 504 static const struct attribute_group *cxl_memdev_attribute_groups[] = { 505 &cxl_memdev_attribute_group, 506 &cxl_memdev_ram_attribute_group, 507 &cxl_memdev_pmem_attribute_group, 508 &cxl_memdev_security_attribute_group, 509 NULL, 510 }; 511 512 static const struct device_type cxl_memdev_type = { 513 .name = "cxl_memdev", 514 .release = cxl_memdev_release, 515 .devnode = cxl_memdev_devnode, 516 .groups = cxl_memdev_attribute_groups, 517 }; 518 519 bool is_cxl_memdev(const struct device *dev) 520 { 521 return dev->type == &cxl_memdev_type; 522 } 523 EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, CXL); 524 525 /** 526 * set_exclusive_cxl_commands() - atomically disable user cxl commands 527 * @mds: The device state to operate on 528 * @cmds: bitmap of commands to mark exclusive 529 * 530 * Grab the cxl_memdev_rwsem in write mode to flush in-flight 531 * invocations of the ioctl path and then disable future execution of 532 * commands with the command ids set in @cmds. 533 */ 534 void set_exclusive_cxl_commands(struct cxl_memdev_state *mds, 535 unsigned long *cmds) 536 { 537 down_write(&cxl_memdev_rwsem); 538 bitmap_or(mds->exclusive_cmds, mds->exclusive_cmds, cmds, 539 CXL_MEM_COMMAND_ID_MAX); 540 up_write(&cxl_memdev_rwsem); 541 } 542 EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, CXL); 543 544 /** 545 * clear_exclusive_cxl_commands() - atomically enable user cxl commands 546 * @mds: The device state to modify 547 * @cmds: bitmap of commands to mark available for userspace 548 */ 549 void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds, 550 unsigned long *cmds) 551 { 552 down_write(&cxl_memdev_rwsem); 553 bitmap_andnot(mds->exclusive_cmds, mds->exclusive_cmds, cmds, 554 CXL_MEM_COMMAND_ID_MAX); 555 up_write(&cxl_memdev_rwsem); 556 } 557 EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, CXL); 558 559 static void cxl_memdev_security_shutdown(struct device *dev) 560 { 561 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 562 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 563 564 if (mds->security.poll) 565 cancel_delayed_work_sync(&mds->security.poll_dwork); 566 } 567 568 static void cxl_memdev_shutdown(struct device *dev) 569 { 570 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 571 572 down_write(&cxl_memdev_rwsem); 573 cxl_memdev_security_shutdown(dev); 574 cxlmd->cxlds = NULL; 575 up_write(&cxl_memdev_rwsem); 576 } 577 578 static void cxl_memdev_unregister(void *_cxlmd) 579 { 580 struct cxl_memdev *cxlmd = _cxlmd; 581 struct device *dev = &cxlmd->dev; 582 583 cxl_memdev_shutdown(dev); 584 cdev_device_del(&cxlmd->cdev, dev); 585 put_device(dev); 586 } 587 588 static void detach_memdev(struct work_struct *work) 589 { 590 struct cxl_memdev *cxlmd; 591 592 cxlmd = container_of(work, typeof(*cxlmd), detach_work); 593 device_release_driver(&cxlmd->dev); 594 put_device(&cxlmd->dev); 595 } 596 597 static struct lock_class_key cxl_memdev_key; 598 599 static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds, 600 const struct file_operations *fops) 601 { 602 struct cxl_memdev *cxlmd; 603 struct device *dev; 604 struct cdev *cdev; 605 int rc; 606 607 cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL); 608 if (!cxlmd) 609 return ERR_PTR(-ENOMEM); 610 611 rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL); 612 if (rc < 0) 613 goto err; 614 cxlmd->id = rc; 615 cxlmd->depth = -1; 616 617 dev = &cxlmd->dev; 618 device_initialize(dev); 619 lockdep_set_class(&dev->mutex, &cxl_memdev_key); 620 dev->parent = cxlds->dev; 621 dev->bus = &cxl_bus_type; 622 dev->devt = MKDEV(cxl_mem_major, cxlmd->id); 623 dev->type = &cxl_memdev_type; 624 device_set_pm_not_required(dev); 625 INIT_WORK(&cxlmd->detach_work, detach_memdev); 626 627 cdev = &cxlmd->cdev; 628 cdev_init(cdev, fops); 629 return cxlmd; 630 631 err: 632 kfree(cxlmd); 633 return ERR_PTR(rc); 634 } 635 636 static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, 637 unsigned long arg) 638 { 639 switch (cmd) { 640 case CXL_MEM_QUERY_COMMANDS: 641 return cxl_query_cmd(cxlmd, (void __user *)arg); 642 case CXL_MEM_SEND_COMMAND: 643 return cxl_send_cmd(cxlmd, (void __user *)arg); 644 default: 645 return -ENOTTY; 646 } 647 } 648 649 static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, 650 unsigned long arg) 651 { 652 struct cxl_memdev *cxlmd = file->private_data; 653 struct cxl_dev_state *cxlds; 654 int rc = -ENXIO; 655 656 down_read(&cxl_memdev_rwsem); 657 cxlds = cxlmd->cxlds; 658 if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM) 659 rc = __cxl_memdev_ioctl(cxlmd, cmd, arg); 660 up_read(&cxl_memdev_rwsem); 661 662 return rc; 663 } 664 665 static int cxl_memdev_open(struct inode *inode, struct file *file) 666 { 667 struct cxl_memdev *cxlmd = 668 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 669 670 get_device(&cxlmd->dev); 671 file->private_data = cxlmd; 672 673 return 0; 674 } 675 676 static int cxl_memdev_release_file(struct inode *inode, struct file *file) 677 { 678 struct cxl_memdev *cxlmd = 679 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 680 681 put_device(&cxlmd->dev); 682 683 return 0; 684 } 685 686 /** 687 * cxl_mem_get_fw_info - Get Firmware info 688 * @mds: The device data for the operation 689 * 690 * Retrieve firmware info for the device specified. 691 * 692 * Return: 0 if no error: or the result of the mailbox command. 693 * 694 * See CXL-3.0 8.2.9.3.1 Get FW Info 695 */ 696 static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds) 697 { 698 struct cxl_mbox_get_fw_info info; 699 struct cxl_mbox_cmd mbox_cmd; 700 int rc; 701 702 mbox_cmd = (struct cxl_mbox_cmd) { 703 .opcode = CXL_MBOX_OP_GET_FW_INFO, 704 .size_out = sizeof(info), 705 .payload_out = &info, 706 }; 707 708 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 709 if (rc < 0) 710 return rc; 711 712 mds->fw.num_slots = info.num_slots; 713 mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK, 714 info.slot_info); 715 716 return 0; 717 } 718 719 /** 720 * cxl_mem_activate_fw - Activate Firmware 721 * @mds: The device data for the operation 722 * @slot: slot number to activate 723 * 724 * Activate firmware in a given slot for the device specified. 725 * 726 * Return: 0 if no error: or the result of the mailbox command. 727 * 728 * See CXL-3.0 8.2.9.3.3 Activate FW 729 */ 730 static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot) 731 { 732 struct cxl_mbox_activate_fw activate; 733 struct cxl_mbox_cmd mbox_cmd; 734 735 if (slot == 0 || slot > mds->fw.num_slots) 736 return -EINVAL; 737 738 mbox_cmd = (struct cxl_mbox_cmd) { 739 .opcode = CXL_MBOX_OP_ACTIVATE_FW, 740 .size_in = sizeof(activate), 741 .payload_in = &activate, 742 }; 743 744 /* Only offline activation supported for now */ 745 activate.action = CXL_FW_ACTIVATE_OFFLINE; 746 activate.slot = slot; 747 748 return cxl_internal_send_cmd(mds, &mbox_cmd); 749 } 750 751 /** 752 * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer 753 * @mds: The device data for the operation 754 * 755 * Abort an in-progress firmware transfer for the device specified. 756 * 757 * Return: 0 if no error: or the result of the mailbox command. 758 * 759 * See CXL-3.0 8.2.9.3.2 Transfer FW 760 */ 761 static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds) 762 { 763 struct cxl_mbox_transfer_fw *transfer; 764 struct cxl_mbox_cmd mbox_cmd; 765 int rc; 766 767 transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL); 768 if (!transfer) 769 return -ENOMEM; 770 771 /* Set a 1s poll interval and a total wait time of 30s */ 772 mbox_cmd = (struct cxl_mbox_cmd) { 773 .opcode = CXL_MBOX_OP_TRANSFER_FW, 774 .size_in = sizeof(*transfer), 775 .payload_in = transfer, 776 .poll_interval_ms = 1000, 777 .poll_count = 30, 778 }; 779 780 transfer->action = CXL_FW_TRANSFER_ACTION_ABORT; 781 782 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 783 kfree(transfer); 784 return rc; 785 } 786 787 static void cxl_fw_cleanup(struct fw_upload *fwl) 788 { 789 struct cxl_memdev_state *mds = fwl->dd_handle; 790 791 mds->fw.next_slot = 0; 792 } 793 794 static int cxl_fw_do_cancel(struct fw_upload *fwl) 795 { 796 struct cxl_memdev_state *mds = fwl->dd_handle; 797 struct cxl_dev_state *cxlds = &mds->cxlds; 798 struct cxl_memdev *cxlmd = cxlds->cxlmd; 799 int rc; 800 801 rc = cxl_mem_abort_fw_xfer(mds); 802 if (rc < 0) 803 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc); 804 805 return FW_UPLOAD_ERR_CANCELED; 806 } 807 808 static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data, 809 u32 size) 810 { 811 struct cxl_memdev_state *mds = fwl->dd_handle; 812 struct cxl_mbox_transfer_fw *transfer; 813 814 if (!size) 815 return FW_UPLOAD_ERR_INVALID_SIZE; 816 817 mds->fw.oneshot = struct_size(transfer, data, size) < 818 mds->payload_size; 819 820 if (cxl_mem_get_fw_info(mds)) 821 return FW_UPLOAD_ERR_HW_ERROR; 822 823 /* 824 * So far no state has been changed, hence no other cleanup is 825 * necessary. Simply return the cancelled status. 826 */ 827 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 828 return FW_UPLOAD_ERR_CANCELED; 829 830 return FW_UPLOAD_ERR_NONE; 831 } 832 833 static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data, 834 u32 offset, u32 size, u32 *written) 835 { 836 struct cxl_memdev_state *mds = fwl->dd_handle; 837 struct cxl_dev_state *cxlds = &mds->cxlds; 838 struct cxl_memdev *cxlmd = cxlds->cxlmd; 839 struct cxl_mbox_transfer_fw *transfer; 840 struct cxl_mbox_cmd mbox_cmd; 841 u32 cur_size, remaining; 842 size_t size_in; 843 int rc; 844 845 *written = 0; 846 847 /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */ 848 if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) { 849 dev_err(&cxlmd->dev, 850 "misaligned offset for FW transfer slice (%u)\n", 851 offset); 852 return FW_UPLOAD_ERR_RW_ERROR; 853 } 854 855 /* 856 * Pick transfer size based on mds->payload_size @size must bw 128-byte 857 * aligned, ->payload_size is a power of 2 starting at 256 bytes, and 858 * sizeof(*transfer) is 128. These constraints imply that @cur_size 859 * will always be 128b aligned. 860 */ 861 cur_size = min_t(size_t, size, mds->payload_size - sizeof(*transfer)); 862 863 remaining = size - cur_size; 864 size_in = struct_size(transfer, data, cur_size); 865 866 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 867 return cxl_fw_do_cancel(fwl); 868 869 /* 870 * Slot numbers are 1-indexed 871 * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1') 872 * Check for rollover using modulo, and 1-index it by adding 1 873 */ 874 mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1; 875 876 /* Do the transfer via mailbox cmd */ 877 transfer = kzalloc(size_in, GFP_KERNEL); 878 if (!transfer) 879 return FW_UPLOAD_ERR_RW_ERROR; 880 881 transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT); 882 memcpy(transfer->data, data + offset, cur_size); 883 if (mds->fw.oneshot) { 884 transfer->action = CXL_FW_TRANSFER_ACTION_FULL; 885 transfer->slot = mds->fw.next_slot; 886 } else { 887 if (offset == 0) { 888 transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE; 889 } else if (remaining == 0) { 890 transfer->action = CXL_FW_TRANSFER_ACTION_END; 891 transfer->slot = mds->fw.next_slot; 892 } else { 893 transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE; 894 } 895 } 896 897 mbox_cmd = (struct cxl_mbox_cmd) { 898 .opcode = CXL_MBOX_OP_TRANSFER_FW, 899 .size_in = size_in, 900 .payload_in = transfer, 901 .poll_interval_ms = 1000, 902 .poll_count = 30, 903 }; 904 905 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 906 if (rc < 0) { 907 rc = FW_UPLOAD_ERR_RW_ERROR; 908 goto out_free; 909 } 910 911 *written = cur_size; 912 913 /* Activate FW if oneshot or if the last slice was written */ 914 if (mds->fw.oneshot || remaining == 0) { 915 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n", 916 mds->fw.next_slot); 917 rc = cxl_mem_activate_fw(mds, mds->fw.next_slot); 918 if (rc < 0) { 919 dev_err(&cxlmd->dev, "Error activating firmware: %d\n", 920 rc); 921 rc = FW_UPLOAD_ERR_HW_ERROR; 922 goto out_free; 923 } 924 } 925 926 rc = FW_UPLOAD_ERR_NONE; 927 928 out_free: 929 kfree(transfer); 930 return rc; 931 } 932 933 static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl) 934 { 935 struct cxl_memdev_state *mds = fwl->dd_handle; 936 937 /* 938 * cxl_internal_send_cmd() handles background operations synchronously. 939 * No need to wait for completions here - any errors would've been 940 * reported and handled during the ->write() call(s). 941 * Just check if a cancel request was received, and return success. 942 */ 943 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 944 return cxl_fw_do_cancel(fwl); 945 946 return FW_UPLOAD_ERR_NONE; 947 } 948 949 static void cxl_fw_cancel(struct fw_upload *fwl) 950 { 951 struct cxl_memdev_state *mds = fwl->dd_handle; 952 953 set_bit(CXL_FW_CANCEL, mds->fw.state); 954 } 955 956 static const struct fw_upload_ops cxl_memdev_fw_ops = { 957 .prepare = cxl_fw_prepare, 958 .write = cxl_fw_write, 959 .poll_complete = cxl_fw_poll_complete, 960 .cancel = cxl_fw_cancel, 961 .cleanup = cxl_fw_cleanup, 962 }; 963 964 static void devm_cxl_remove_fw_upload(void *fwl) 965 { 966 firmware_upload_unregister(fwl); 967 } 968 969 int cxl_memdev_setup_fw_upload(struct cxl_memdev_state *mds) 970 { 971 struct cxl_dev_state *cxlds = &mds->cxlds; 972 struct device *dev = &cxlds->cxlmd->dev; 973 struct fw_upload *fwl; 974 int rc; 975 976 if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, mds->enabled_cmds)) 977 return 0; 978 979 fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev), 980 &cxl_memdev_fw_ops, mds); 981 if (IS_ERR(fwl)) 982 return dev_err_probe(dev, PTR_ERR(fwl), 983 "Failed to register firmware loader\n"); 984 985 rc = devm_add_action_or_reset(cxlds->dev, devm_cxl_remove_fw_upload, 986 fwl); 987 if (rc) 988 dev_err(dev, 989 "Failed to add firmware loader remove action: %d\n", 990 rc); 991 992 return rc; 993 } 994 EXPORT_SYMBOL_NS_GPL(cxl_memdev_setup_fw_upload, CXL); 995 996 static const struct file_operations cxl_memdev_fops = { 997 .owner = THIS_MODULE, 998 .unlocked_ioctl = cxl_memdev_ioctl, 999 .open = cxl_memdev_open, 1000 .release = cxl_memdev_release_file, 1001 .compat_ioctl = compat_ptr_ioctl, 1002 .llseek = noop_llseek, 1003 }; 1004 1005 static void put_sanitize(void *data) 1006 { 1007 struct cxl_memdev_state *mds = data; 1008 1009 sysfs_put(mds->security.sanitize_node); 1010 } 1011 1012 static int cxl_memdev_security_init(struct cxl_memdev *cxlmd) 1013 { 1014 struct cxl_dev_state *cxlds = cxlmd->cxlds; 1015 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 1016 struct device *dev = &cxlmd->dev; 1017 struct kernfs_node *sec; 1018 1019 sec = sysfs_get_dirent(dev->kobj.sd, "security"); 1020 if (!sec) { 1021 dev_err(dev, "sysfs_get_dirent 'security' failed\n"); 1022 return -ENODEV; 1023 } 1024 mds->security.sanitize_node = sysfs_get_dirent(sec, "state"); 1025 sysfs_put(sec); 1026 if (!mds->security.sanitize_node) { 1027 dev_err(dev, "sysfs_get_dirent 'state' failed\n"); 1028 return -ENODEV; 1029 } 1030 1031 return devm_add_action_or_reset(cxlds->dev, put_sanitize, mds); 1032 } 1033 1034 struct cxl_memdev *devm_cxl_add_memdev(struct cxl_dev_state *cxlds) 1035 { 1036 struct cxl_memdev *cxlmd; 1037 struct device *dev; 1038 struct cdev *cdev; 1039 int rc; 1040 1041 cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops); 1042 if (IS_ERR(cxlmd)) 1043 return cxlmd; 1044 1045 dev = &cxlmd->dev; 1046 rc = dev_set_name(dev, "mem%d", cxlmd->id); 1047 if (rc) 1048 goto err; 1049 1050 /* 1051 * Activate ioctl operations, no cxl_memdev_rwsem manipulation 1052 * needed as this is ordered with cdev_add() publishing the device. 1053 */ 1054 cxlmd->cxlds = cxlds; 1055 cxlds->cxlmd = cxlmd; 1056 1057 cdev = &cxlmd->cdev; 1058 rc = cdev_device_add(cdev, dev); 1059 if (rc) 1060 goto err; 1061 1062 rc = cxl_memdev_security_init(cxlmd); 1063 if (rc) 1064 goto err; 1065 1066 rc = devm_add_action_or_reset(cxlds->dev, cxl_memdev_unregister, cxlmd); 1067 if (rc) 1068 return ERR_PTR(rc); 1069 return cxlmd; 1070 1071 err: 1072 /* 1073 * The cdev was briefly live, shutdown any ioctl operations that 1074 * saw that state. 1075 */ 1076 cxl_memdev_shutdown(dev); 1077 put_device(dev); 1078 return ERR_PTR(rc); 1079 } 1080 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL); 1081 1082 __init int cxl_memdev_init(void) 1083 { 1084 dev_t devt; 1085 int rc; 1086 1087 rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); 1088 if (rc) 1089 return rc; 1090 1091 cxl_mem_major = MAJOR(devt); 1092 1093 return 0; 1094 } 1095 1096 void cxl_memdev_exit(void) 1097 { 1098 unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS); 1099 } 1100