1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2020 Intel Corporation. */ 3 4 #include <linux/io-64-nonatomic-lo-hi.h> 5 #include <linux/firmware.h> 6 #include <linux/device.h> 7 #include <linux/slab.h> 8 #include <linux/idr.h> 9 #include <linux/pci.h> 10 #include <cxlmem.h> 11 #include "trace.h" 12 #include "core.h" 13 14 static DECLARE_RWSEM(cxl_memdev_rwsem); 15 16 /* 17 * An entire PCI topology full of devices should be enough for any 18 * config 19 */ 20 #define CXL_MEM_MAX_DEVS 65536 21 22 static int cxl_mem_major; 23 static DEFINE_IDA(cxl_memdev_ida); 24 25 static void cxl_memdev_release(struct device *dev) 26 { 27 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 28 29 ida_free(&cxl_memdev_ida, cxlmd->id); 30 kfree(cxlmd); 31 } 32 33 static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid, 34 kgid_t *gid) 35 { 36 return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); 37 } 38 39 static ssize_t firmware_version_show(struct device *dev, 40 struct device_attribute *attr, char *buf) 41 { 42 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 43 struct cxl_dev_state *cxlds = cxlmd->cxlds; 44 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 45 46 if (!mds) 47 return sysfs_emit(buf, "\n"); 48 return sysfs_emit(buf, "%.16s\n", mds->firmware_version); 49 } 50 static DEVICE_ATTR_RO(firmware_version); 51 52 static ssize_t payload_max_show(struct device *dev, 53 struct device_attribute *attr, char *buf) 54 { 55 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 56 struct cxl_dev_state *cxlds = cxlmd->cxlds; 57 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 58 59 if (!mds) 60 return sysfs_emit(buf, "\n"); 61 return sysfs_emit(buf, "%zu\n", mds->payload_size); 62 } 63 static DEVICE_ATTR_RO(payload_max); 64 65 static ssize_t label_storage_size_show(struct device *dev, 66 struct device_attribute *attr, char *buf) 67 { 68 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 69 struct cxl_dev_state *cxlds = cxlmd->cxlds; 70 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 71 72 if (!mds) 73 return sysfs_emit(buf, "\n"); 74 return sysfs_emit(buf, "%zu\n", mds->lsa_size); 75 } 76 static DEVICE_ATTR_RO(label_storage_size); 77 78 static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, 79 char *buf) 80 { 81 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 82 struct cxl_dev_state *cxlds = cxlmd->cxlds; 83 unsigned long long len = resource_size(&cxlds->ram_res); 84 85 return sysfs_emit(buf, "%#llx\n", len); 86 } 87 88 static struct device_attribute dev_attr_ram_size = 89 __ATTR(size, 0444, ram_size_show, NULL); 90 91 static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, 92 char *buf) 93 { 94 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 95 struct cxl_dev_state *cxlds = cxlmd->cxlds; 96 unsigned long long len = resource_size(&cxlds->pmem_res); 97 98 return sysfs_emit(buf, "%#llx\n", len); 99 } 100 101 static struct device_attribute dev_attr_pmem_size = 102 __ATTR(size, 0444, pmem_size_show, NULL); 103 104 static ssize_t serial_show(struct device *dev, struct device_attribute *attr, 105 char *buf) 106 { 107 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 108 struct cxl_dev_state *cxlds = cxlmd->cxlds; 109 110 return sysfs_emit(buf, "%#llx\n", cxlds->serial); 111 } 112 static DEVICE_ATTR_RO(serial); 113 114 static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, 115 char *buf) 116 { 117 return sprintf(buf, "%d\n", dev_to_node(dev)); 118 } 119 static DEVICE_ATTR_RO(numa_node); 120 121 static ssize_t security_state_show(struct device *dev, 122 struct device_attribute *attr, 123 char *buf) 124 { 125 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 126 struct cxl_dev_state *cxlds = cxlmd->cxlds; 127 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 128 unsigned long state = mds->security.state; 129 int rc = 0; 130 131 /* sync with latest submission state */ 132 mutex_lock(&mds->mbox_mutex); 133 if (mds->security.sanitize_active) 134 rc = sysfs_emit(buf, "sanitize\n"); 135 mutex_unlock(&mds->mbox_mutex); 136 if (rc) 137 return rc; 138 139 if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET)) 140 return sysfs_emit(buf, "disabled\n"); 141 if (state & CXL_PMEM_SEC_STATE_FROZEN || 142 state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT || 143 state & CXL_PMEM_SEC_STATE_USER_PLIMIT) 144 return sysfs_emit(buf, "frozen\n"); 145 if (state & CXL_PMEM_SEC_STATE_LOCKED) 146 return sysfs_emit(buf, "locked\n"); 147 else 148 return sysfs_emit(buf, "unlocked\n"); 149 } 150 static struct device_attribute dev_attr_security_state = 151 __ATTR(state, 0444, security_state_show, NULL); 152 153 static ssize_t security_sanitize_store(struct device *dev, 154 struct device_attribute *attr, 155 const char *buf, size_t len) 156 { 157 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 158 bool sanitize; 159 ssize_t rc; 160 161 if (kstrtobool(buf, &sanitize) || !sanitize) 162 return -EINVAL; 163 164 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE); 165 if (rc) 166 return rc; 167 168 return len; 169 } 170 static struct device_attribute dev_attr_security_sanitize = 171 __ATTR(sanitize, 0200, NULL, security_sanitize_store); 172 173 static ssize_t security_erase_store(struct device *dev, 174 struct device_attribute *attr, 175 const char *buf, size_t len) 176 { 177 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 178 ssize_t rc; 179 bool erase; 180 181 if (kstrtobool(buf, &erase) || !erase) 182 return -EINVAL; 183 184 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE); 185 if (rc) 186 return rc; 187 188 return len; 189 } 190 static struct device_attribute dev_attr_security_erase = 191 __ATTR(erase, 0200, NULL, security_erase_store); 192 193 static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd) 194 { 195 struct cxl_dev_state *cxlds = cxlmd->cxlds; 196 u64 offset, length; 197 int rc = 0; 198 199 /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */ 200 if (resource_size(&cxlds->pmem_res)) { 201 offset = cxlds->pmem_res.start; 202 length = resource_size(&cxlds->pmem_res); 203 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 204 if (rc) 205 return rc; 206 } 207 if (resource_size(&cxlds->ram_res)) { 208 offset = cxlds->ram_res.start; 209 length = resource_size(&cxlds->ram_res); 210 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 211 /* 212 * Invalid Physical Address is not an error for 213 * volatile addresses. Device support is optional. 214 */ 215 if (rc == -EFAULT) 216 rc = 0; 217 } 218 return rc; 219 } 220 221 int cxl_trigger_poison_list(struct cxl_memdev *cxlmd) 222 { 223 struct cxl_port *port; 224 int rc; 225 226 port = cxlmd->endpoint; 227 if (!port || !is_cxl_endpoint(port)) 228 return -EINVAL; 229 230 rc = down_read_interruptible(&cxl_dpa_rwsem); 231 if (rc) 232 return rc; 233 234 if (port->commit_end == -1) { 235 /* No regions mapped to this memdev */ 236 rc = cxl_get_poison_by_memdev(cxlmd); 237 } else { 238 /* Regions mapped, collect poison by endpoint */ 239 rc = cxl_get_poison_by_endpoint(port); 240 } 241 up_read(&cxl_dpa_rwsem); 242 243 return rc; 244 } 245 EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, CXL); 246 247 struct cxl_dpa_to_region_context { 248 struct cxl_region *cxlr; 249 u64 dpa; 250 }; 251 252 static int __cxl_dpa_to_region(struct device *dev, void *arg) 253 { 254 struct cxl_dpa_to_region_context *ctx = arg; 255 struct cxl_endpoint_decoder *cxled; 256 u64 dpa = ctx->dpa; 257 258 if (!is_endpoint_decoder(dev)) 259 return 0; 260 261 cxled = to_cxl_endpoint_decoder(dev); 262 if (!cxled->dpa_res || !resource_size(cxled->dpa_res)) 263 return 0; 264 265 if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start) 266 return 0; 267 268 dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa, 269 dev_name(&cxled->cxld.region->dev)); 270 271 ctx->cxlr = cxled->cxld.region; 272 273 return 1; 274 } 275 276 static struct cxl_region *cxl_dpa_to_region(struct cxl_memdev *cxlmd, u64 dpa) 277 { 278 struct cxl_dpa_to_region_context ctx; 279 struct cxl_port *port; 280 281 ctx = (struct cxl_dpa_to_region_context) { 282 .dpa = dpa, 283 }; 284 port = cxlmd->endpoint; 285 if (port && is_cxl_endpoint(port) && port->commit_end != -1) 286 device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region); 287 288 return ctx.cxlr; 289 } 290 291 static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa) 292 { 293 struct cxl_dev_state *cxlds = cxlmd->cxlds; 294 295 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 296 return 0; 297 298 if (!resource_size(&cxlds->dpa_res)) { 299 dev_dbg(cxlds->dev, "device has no dpa resource\n"); 300 return -EINVAL; 301 } 302 if (dpa < cxlds->dpa_res.start || dpa > cxlds->dpa_res.end) { 303 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n", 304 dpa, &cxlds->dpa_res); 305 return -EINVAL; 306 } 307 if (!IS_ALIGNED(dpa, 64)) { 308 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa); 309 return -EINVAL; 310 } 311 312 return 0; 313 } 314 315 int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa) 316 { 317 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 318 struct cxl_mbox_inject_poison inject; 319 struct cxl_poison_record record; 320 struct cxl_mbox_cmd mbox_cmd; 321 struct cxl_region *cxlr; 322 int rc; 323 324 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 325 return 0; 326 327 rc = down_read_interruptible(&cxl_dpa_rwsem); 328 if (rc) 329 return rc; 330 331 rc = cxl_validate_poison_dpa(cxlmd, dpa); 332 if (rc) 333 goto out; 334 335 inject.address = cpu_to_le64(dpa); 336 mbox_cmd = (struct cxl_mbox_cmd) { 337 .opcode = CXL_MBOX_OP_INJECT_POISON, 338 .size_in = sizeof(inject), 339 .payload_in = &inject, 340 }; 341 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 342 if (rc) 343 goto out; 344 345 cxlr = cxl_dpa_to_region(cxlmd, dpa); 346 if (cxlr) 347 dev_warn_once(mds->cxlds.dev, 348 "poison inject dpa:%#llx region: %s\n", dpa, 349 dev_name(&cxlr->dev)); 350 351 record = (struct cxl_poison_record) { 352 .address = cpu_to_le64(dpa), 353 .length = cpu_to_le32(1), 354 }; 355 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT); 356 out: 357 up_read(&cxl_dpa_rwsem); 358 359 return rc; 360 } 361 EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, CXL); 362 363 int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa) 364 { 365 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 366 struct cxl_mbox_clear_poison clear; 367 struct cxl_poison_record record; 368 struct cxl_mbox_cmd mbox_cmd; 369 struct cxl_region *cxlr; 370 int rc; 371 372 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 373 return 0; 374 375 rc = down_read_interruptible(&cxl_dpa_rwsem); 376 if (rc) 377 return rc; 378 379 rc = cxl_validate_poison_dpa(cxlmd, dpa); 380 if (rc) 381 goto out; 382 383 /* 384 * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command 385 * is defined to accept 64 bytes of write-data, along with the 386 * address to clear. This driver uses zeroes as write-data. 387 */ 388 clear = (struct cxl_mbox_clear_poison) { 389 .address = cpu_to_le64(dpa) 390 }; 391 392 mbox_cmd = (struct cxl_mbox_cmd) { 393 .opcode = CXL_MBOX_OP_CLEAR_POISON, 394 .size_in = sizeof(clear), 395 .payload_in = &clear, 396 }; 397 398 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 399 if (rc) 400 goto out; 401 402 cxlr = cxl_dpa_to_region(cxlmd, dpa); 403 if (cxlr) 404 dev_warn_once(mds->cxlds.dev, 405 "poison clear dpa:%#llx region: %s\n", dpa, 406 dev_name(&cxlr->dev)); 407 408 record = (struct cxl_poison_record) { 409 .address = cpu_to_le64(dpa), 410 .length = cpu_to_le32(1), 411 }; 412 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR); 413 out: 414 up_read(&cxl_dpa_rwsem); 415 416 return rc; 417 } 418 EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, CXL); 419 420 static struct attribute *cxl_memdev_attributes[] = { 421 &dev_attr_serial.attr, 422 &dev_attr_firmware_version.attr, 423 &dev_attr_payload_max.attr, 424 &dev_attr_label_storage_size.attr, 425 &dev_attr_numa_node.attr, 426 NULL, 427 }; 428 429 static struct attribute *cxl_memdev_pmem_attributes[] = { 430 &dev_attr_pmem_size.attr, 431 NULL, 432 }; 433 434 static struct attribute *cxl_memdev_ram_attributes[] = { 435 &dev_attr_ram_size.attr, 436 NULL, 437 }; 438 439 static struct attribute *cxl_memdev_security_attributes[] = { 440 &dev_attr_security_state.attr, 441 &dev_attr_security_sanitize.attr, 442 &dev_attr_security_erase.attr, 443 NULL, 444 }; 445 446 static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a, 447 int n) 448 { 449 if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr) 450 return 0; 451 return a->mode; 452 } 453 454 static struct attribute_group cxl_memdev_attribute_group = { 455 .attrs = cxl_memdev_attributes, 456 .is_visible = cxl_memdev_visible, 457 }; 458 459 static struct attribute_group cxl_memdev_ram_attribute_group = { 460 .name = "ram", 461 .attrs = cxl_memdev_ram_attributes, 462 }; 463 464 static struct attribute_group cxl_memdev_pmem_attribute_group = { 465 .name = "pmem", 466 .attrs = cxl_memdev_pmem_attributes, 467 }; 468 469 static umode_t cxl_memdev_security_visible(struct kobject *kobj, 470 struct attribute *a, int n) 471 { 472 struct device *dev = kobj_to_dev(kobj); 473 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 474 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 475 476 if (a == &dev_attr_security_sanitize.attr && 477 !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) 478 return 0; 479 480 if (a == &dev_attr_security_erase.attr && 481 !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds)) 482 return 0; 483 484 return a->mode; 485 } 486 487 static struct attribute_group cxl_memdev_security_attribute_group = { 488 .name = "security", 489 .attrs = cxl_memdev_security_attributes, 490 .is_visible = cxl_memdev_security_visible, 491 }; 492 493 static const struct attribute_group *cxl_memdev_attribute_groups[] = { 494 &cxl_memdev_attribute_group, 495 &cxl_memdev_ram_attribute_group, 496 &cxl_memdev_pmem_attribute_group, 497 &cxl_memdev_security_attribute_group, 498 NULL, 499 }; 500 501 static const struct device_type cxl_memdev_type = { 502 .name = "cxl_memdev", 503 .release = cxl_memdev_release, 504 .devnode = cxl_memdev_devnode, 505 .groups = cxl_memdev_attribute_groups, 506 }; 507 508 bool is_cxl_memdev(const struct device *dev) 509 { 510 return dev->type == &cxl_memdev_type; 511 } 512 EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, CXL); 513 514 /** 515 * set_exclusive_cxl_commands() - atomically disable user cxl commands 516 * @mds: The device state to operate on 517 * @cmds: bitmap of commands to mark exclusive 518 * 519 * Grab the cxl_memdev_rwsem in write mode to flush in-flight 520 * invocations of the ioctl path and then disable future execution of 521 * commands with the command ids set in @cmds. 522 */ 523 void set_exclusive_cxl_commands(struct cxl_memdev_state *mds, 524 unsigned long *cmds) 525 { 526 down_write(&cxl_memdev_rwsem); 527 bitmap_or(mds->exclusive_cmds, mds->exclusive_cmds, cmds, 528 CXL_MEM_COMMAND_ID_MAX); 529 up_write(&cxl_memdev_rwsem); 530 } 531 EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, CXL); 532 533 /** 534 * clear_exclusive_cxl_commands() - atomically enable user cxl commands 535 * @mds: The device state to modify 536 * @cmds: bitmap of commands to mark available for userspace 537 */ 538 void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds, 539 unsigned long *cmds) 540 { 541 down_write(&cxl_memdev_rwsem); 542 bitmap_andnot(mds->exclusive_cmds, mds->exclusive_cmds, cmds, 543 CXL_MEM_COMMAND_ID_MAX); 544 up_write(&cxl_memdev_rwsem); 545 } 546 EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, CXL); 547 548 static void cxl_memdev_shutdown(struct device *dev) 549 { 550 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 551 552 down_write(&cxl_memdev_rwsem); 553 cxlmd->cxlds = NULL; 554 up_write(&cxl_memdev_rwsem); 555 } 556 557 static void cxl_memdev_unregister(void *_cxlmd) 558 { 559 struct cxl_memdev *cxlmd = _cxlmd; 560 struct device *dev = &cxlmd->dev; 561 562 cdev_device_del(&cxlmd->cdev, dev); 563 cxl_memdev_shutdown(dev); 564 put_device(dev); 565 } 566 567 static void detach_memdev(struct work_struct *work) 568 { 569 struct cxl_memdev *cxlmd; 570 571 cxlmd = container_of(work, typeof(*cxlmd), detach_work); 572 device_release_driver(&cxlmd->dev); 573 put_device(&cxlmd->dev); 574 } 575 576 static struct lock_class_key cxl_memdev_key; 577 578 static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds, 579 const struct file_operations *fops) 580 { 581 struct cxl_memdev *cxlmd; 582 struct device *dev; 583 struct cdev *cdev; 584 int rc; 585 586 cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL); 587 if (!cxlmd) 588 return ERR_PTR(-ENOMEM); 589 590 rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL); 591 if (rc < 0) 592 goto err; 593 cxlmd->id = rc; 594 cxlmd->depth = -1; 595 596 dev = &cxlmd->dev; 597 device_initialize(dev); 598 lockdep_set_class(&dev->mutex, &cxl_memdev_key); 599 dev->parent = cxlds->dev; 600 dev->bus = &cxl_bus_type; 601 dev->devt = MKDEV(cxl_mem_major, cxlmd->id); 602 dev->type = &cxl_memdev_type; 603 device_set_pm_not_required(dev); 604 INIT_WORK(&cxlmd->detach_work, detach_memdev); 605 606 cdev = &cxlmd->cdev; 607 cdev_init(cdev, fops); 608 return cxlmd; 609 610 err: 611 kfree(cxlmd); 612 return ERR_PTR(rc); 613 } 614 615 static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, 616 unsigned long arg) 617 { 618 switch (cmd) { 619 case CXL_MEM_QUERY_COMMANDS: 620 return cxl_query_cmd(cxlmd, (void __user *)arg); 621 case CXL_MEM_SEND_COMMAND: 622 return cxl_send_cmd(cxlmd, (void __user *)arg); 623 default: 624 return -ENOTTY; 625 } 626 } 627 628 static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, 629 unsigned long arg) 630 { 631 struct cxl_memdev *cxlmd = file->private_data; 632 struct cxl_dev_state *cxlds; 633 int rc = -ENXIO; 634 635 down_read(&cxl_memdev_rwsem); 636 cxlds = cxlmd->cxlds; 637 if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM) 638 rc = __cxl_memdev_ioctl(cxlmd, cmd, arg); 639 up_read(&cxl_memdev_rwsem); 640 641 return rc; 642 } 643 644 static int cxl_memdev_open(struct inode *inode, struct file *file) 645 { 646 struct cxl_memdev *cxlmd = 647 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 648 649 get_device(&cxlmd->dev); 650 file->private_data = cxlmd; 651 652 return 0; 653 } 654 655 static int cxl_memdev_release_file(struct inode *inode, struct file *file) 656 { 657 struct cxl_memdev *cxlmd = 658 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 659 660 put_device(&cxlmd->dev); 661 662 return 0; 663 } 664 665 /** 666 * cxl_mem_get_fw_info - Get Firmware info 667 * @mds: The device data for the operation 668 * 669 * Retrieve firmware info for the device specified. 670 * 671 * Return: 0 if no error: or the result of the mailbox command. 672 * 673 * See CXL-3.0 8.2.9.3.1 Get FW Info 674 */ 675 static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds) 676 { 677 struct cxl_mbox_get_fw_info info; 678 struct cxl_mbox_cmd mbox_cmd; 679 int rc; 680 681 mbox_cmd = (struct cxl_mbox_cmd) { 682 .opcode = CXL_MBOX_OP_GET_FW_INFO, 683 .size_out = sizeof(info), 684 .payload_out = &info, 685 }; 686 687 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 688 if (rc < 0) 689 return rc; 690 691 mds->fw.num_slots = info.num_slots; 692 mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK, 693 info.slot_info); 694 695 return 0; 696 } 697 698 /** 699 * cxl_mem_activate_fw - Activate Firmware 700 * @mds: The device data for the operation 701 * @slot: slot number to activate 702 * 703 * Activate firmware in a given slot for the device specified. 704 * 705 * Return: 0 if no error: or the result of the mailbox command. 706 * 707 * See CXL-3.0 8.2.9.3.3 Activate FW 708 */ 709 static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot) 710 { 711 struct cxl_mbox_activate_fw activate; 712 struct cxl_mbox_cmd mbox_cmd; 713 714 if (slot == 0 || slot > mds->fw.num_slots) 715 return -EINVAL; 716 717 mbox_cmd = (struct cxl_mbox_cmd) { 718 .opcode = CXL_MBOX_OP_ACTIVATE_FW, 719 .size_in = sizeof(activate), 720 .payload_in = &activate, 721 }; 722 723 /* Only offline activation supported for now */ 724 activate.action = CXL_FW_ACTIVATE_OFFLINE; 725 activate.slot = slot; 726 727 return cxl_internal_send_cmd(mds, &mbox_cmd); 728 } 729 730 /** 731 * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer 732 * @mds: The device data for the operation 733 * 734 * Abort an in-progress firmware transfer for the device specified. 735 * 736 * Return: 0 if no error: or the result of the mailbox command. 737 * 738 * See CXL-3.0 8.2.9.3.2 Transfer FW 739 */ 740 static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds) 741 { 742 struct cxl_mbox_transfer_fw *transfer; 743 struct cxl_mbox_cmd mbox_cmd; 744 int rc; 745 746 transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL); 747 if (!transfer) 748 return -ENOMEM; 749 750 /* Set a 1s poll interval and a total wait time of 30s */ 751 mbox_cmd = (struct cxl_mbox_cmd) { 752 .opcode = CXL_MBOX_OP_TRANSFER_FW, 753 .size_in = sizeof(*transfer), 754 .payload_in = transfer, 755 .poll_interval_ms = 1000, 756 .poll_count = 30, 757 }; 758 759 transfer->action = CXL_FW_TRANSFER_ACTION_ABORT; 760 761 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 762 kfree(transfer); 763 return rc; 764 } 765 766 static void cxl_fw_cleanup(struct fw_upload *fwl) 767 { 768 struct cxl_memdev_state *mds = fwl->dd_handle; 769 770 mds->fw.next_slot = 0; 771 } 772 773 static int cxl_fw_do_cancel(struct fw_upload *fwl) 774 { 775 struct cxl_memdev_state *mds = fwl->dd_handle; 776 struct cxl_dev_state *cxlds = &mds->cxlds; 777 struct cxl_memdev *cxlmd = cxlds->cxlmd; 778 int rc; 779 780 rc = cxl_mem_abort_fw_xfer(mds); 781 if (rc < 0) 782 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc); 783 784 return FW_UPLOAD_ERR_CANCELED; 785 } 786 787 static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data, 788 u32 size) 789 { 790 struct cxl_memdev_state *mds = fwl->dd_handle; 791 struct cxl_mbox_transfer_fw *transfer; 792 793 if (!size) 794 return FW_UPLOAD_ERR_INVALID_SIZE; 795 796 mds->fw.oneshot = struct_size(transfer, data, size) < 797 mds->payload_size; 798 799 if (cxl_mem_get_fw_info(mds)) 800 return FW_UPLOAD_ERR_HW_ERROR; 801 802 /* 803 * So far no state has been changed, hence no other cleanup is 804 * necessary. Simply return the cancelled status. 805 */ 806 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 807 return FW_UPLOAD_ERR_CANCELED; 808 809 return FW_UPLOAD_ERR_NONE; 810 } 811 812 static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data, 813 u32 offset, u32 size, u32 *written) 814 { 815 struct cxl_memdev_state *mds = fwl->dd_handle; 816 struct cxl_dev_state *cxlds = &mds->cxlds; 817 struct cxl_memdev *cxlmd = cxlds->cxlmd; 818 struct cxl_mbox_transfer_fw *transfer; 819 struct cxl_mbox_cmd mbox_cmd; 820 u32 cur_size, remaining; 821 size_t size_in; 822 int rc; 823 824 *written = 0; 825 826 /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */ 827 if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) { 828 dev_err(&cxlmd->dev, 829 "misaligned offset for FW transfer slice (%u)\n", 830 offset); 831 return FW_UPLOAD_ERR_RW_ERROR; 832 } 833 834 /* 835 * Pick transfer size based on mds->payload_size @size must bw 128-byte 836 * aligned, ->payload_size is a power of 2 starting at 256 bytes, and 837 * sizeof(*transfer) is 128. These constraints imply that @cur_size 838 * will always be 128b aligned. 839 */ 840 cur_size = min_t(size_t, size, mds->payload_size - sizeof(*transfer)); 841 842 remaining = size - cur_size; 843 size_in = struct_size(transfer, data, cur_size); 844 845 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 846 return cxl_fw_do_cancel(fwl); 847 848 /* 849 * Slot numbers are 1-indexed 850 * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1') 851 * Check for rollover using modulo, and 1-index it by adding 1 852 */ 853 mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1; 854 855 /* Do the transfer via mailbox cmd */ 856 transfer = kzalloc(size_in, GFP_KERNEL); 857 if (!transfer) 858 return FW_UPLOAD_ERR_RW_ERROR; 859 860 transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT); 861 memcpy(transfer->data, data + offset, cur_size); 862 if (mds->fw.oneshot) { 863 transfer->action = CXL_FW_TRANSFER_ACTION_FULL; 864 transfer->slot = mds->fw.next_slot; 865 } else { 866 if (offset == 0) { 867 transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE; 868 } else if (remaining == 0) { 869 transfer->action = CXL_FW_TRANSFER_ACTION_END; 870 transfer->slot = mds->fw.next_slot; 871 } else { 872 transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE; 873 } 874 } 875 876 mbox_cmd = (struct cxl_mbox_cmd) { 877 .opcode = CXL_MBOX_OP_TRANSFER_FW, 878 .size_in = size_in, 879 .payload_in = transfer, 880 .poll_interval_ms = 1000, 881 .poll_count = 30, 882 }; 883 884 rc = cxl_internal_send_cmd(mds, &mbox_cmd); 885 if (rc < 0) { 886 rc = FW_UPLOAD_ERR_RW_ERROR; 887 goto out_free; 888 } 889 890 *written = cur_size; 891 892 /* Activate FW if oneshot or if the last slice was written */ 893 if (mds->fw.oneshot || remaining == 0) { 894 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n", 895 mds->fw.next_slot); 896 rc = cxl_mem_activate_fw(mds, mds->fw.next_slot); 897 if (rc < 0) { 898 dev_err(&cxlmd->dev, "Error activating firmware: %d\n", 899 rc); 900 rc = FW_UPLOAD_ERR_HW_ERROR; 901 goto out_free; 902 } 903 } 904 905 rc = FW_UPLOAD_ERR_NONE; 906 907 out_free: 908 kfree(transfer); 909 return rc; 910 } 911 912 static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl) 913 { 914 struct cxl_memdev_state *mds = fwl->dd_handle; 915 916 /* 917 * cxl_internal_send_cmd() handles background operations synchronously. 918 * No need to wait for completions here - any errors would've been 919 * reported and handled during the ->write() call(s). 920 * Just check if a cancel request was received, and return success. 921 */ 922 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 923 return cxl_fw_do_cancel(fwl); 924 925 return FW_UPLOAD_ERR_NONE; 926 } 927 928 static void cxl_fw_cancel(struct fw_upload *fwl) 929 { 930 struct cxl_memdev_state *mds = fwl->dd_handle; 931 932 set_bit(CXL_FW_CANCEL, mds->fw.state); 933 } 934 935 static const struct fw_upload_ops cxl_memdev_fw_ops = { 936 .prepare = cxl_fw_prepare, 937 .write = cxl_fw_write, 938 .poll_complete = cxl_fw_poll_complete, 939 .cancel = cxl_fw_cancel, 940 .cleanup = cxl_fw_cleanup, 941 }; 942 943 static void cxl_remove_fw_upload(void *fwl) 944 { 945 firmware_upload_unregister(fwl); 946 } 947 948 int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds) 949 { 950 struct cxl_dev_state *cxlds = &mds->cxlds; 951 struct device *dev = &cxlds->cxlmd->dev; 952 struct fw_upload *fwl; 953 954 if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, mds->enabled_cmds)) 955 return 0; 956 957 fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev), 958 &cxl_memdev_fw_ops, mds); 959 if (IS_ERR(fwl)) 960 return PTR_ERR(fwl); 961 return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl); 962 } 963 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, CXL); 964 965 static const struct file_operations cxl_memdev_fops = { 966 .owner = THIS_MODULE, 967 .unlocked_ioctl = cxl_memdev_ioctl, 968 .open = cxl_memdev_open, 969 .release = cxl_memdev_release_file, 970 .compat_ioctl = compat_ptr_ioctl, 971 .llseek = noop_llseek, 972 }; 973 974 struct cxl_memdev *devm_cxl_add_memdev(struct device *host, 975 struct cxl_dev_state *cxlds) 976 { 977 struct cxl_memdev *cxlmd; 978 struct device *dev; 979 struct cdev *cdev; 980 int rc; 981 982 cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops); 983 if (IS_ERR(cxlmd)) 984 return cxlmd; 985 986 dev = &cxlmd->dev; 987 rc = dev_set_name(dev, "mem%d", cxlmd->id); 988 if (rc) 989 goto err; 990 991 /* 992 * Activate ioctl operations, no cxl_memdev_rwsem manipulation 993 * needed as this is ordered with cdev_add() publishing the device. 994 */ 995 cxlmd->cxlds = cxlds; 996 cxlds->cxlmd = cxlmd; 997 998 cdev = &cxlmd->cdev; 999 rc = cdev_device_add(cdev, dev); 1000 if (rc) 1001 goto err; 1002 1003 rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd); 1004 if (rc) 1005 return ERR_PTR(rc); 1006 return cxlmd; 1007 1008 err: 1009 /* 1010 * The cdev was briefly live, shutdown any ioctl operations that 1011 * saw that state. 1012 */ 1013 cxl_memdev_shutdown(dev); 1014 put_device(dev); 1015 return ERR_PTR(rc); 1016 } 1017 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL); 1018 1019 static void sanitize_teardown_notifier(void *data) 1020 { 1021 struct cxl_memdev_state *mds = data; 1022 struct kernfs_node *state; 1023 1024 /* 1025 * Prevent new irq triggered invocations of the workqueue and 1026 * flush inflight invocations. 1027 */ 1028 mutex_lock(&mds->mbox_mutex); 1029 state = mds->security.sanitize_node; 1030 mds->security.sanitize_node = NULL; 1031 mutex_unlock(&mds->mbox_mutex); 1032 1033 cancel_delayed_work_sync(&mds->security.poll_dwork); 1034 sysfs_put(state); 1035 } 1036 1037 int devm_cxl_sanitize_setup_notifier(struct device *host, 1038 struct cxl_memdev *cxlmd) 1039 { 1040 struct cxl_dev_state *cxlds = cxlmd->cxlds; 1041 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 1042 struct kernfs_node *sec; 1043 1044 if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) 1045 return 0; 1046 1047 /* 1048 * Note, the expectation is that @cxlmd would have failed to be 1049 * created if these sysfs_get_dirent calls fail. 1050 */ 1051 sec = sysfs_get_dirent(cxlmd->dev.kobj.sd, "security"); 1052 if (!sec) 1053 return -ENOENT; 1054 mds->security.sanitize_node = sysfs_get_dirent(sec, "state"); 1055 sysfs_put(sec); 1056 if (!mds->security.sanitize_node) 1057 return -ENOENT; 1058 1059 return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds); 1060 } 1061 EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, CXL); 1062 1063 __init int cxl_memdev_init(void) 1064 { 1065 dev_t devt; 1066 int rc; 1067 1068 rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); 1069 if (rc) 1070 return rc; 1071 1072 cxl_mem_major = MAJOR(devt); 1073 1074 return 0; 1075 } 1076 1077 void cxl_memdev_exit(void) 1078 { 1079 unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS); 1080 } 1081