1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2020 Intel Corporation. */ 3 4 #include <linux/firmware.h> 5 #include <linux/device.h> 6 #include <linux/slab.h> 7 #include <linux/idr.h> 8 #include <linux/pci.h> 9 #include <cxlmem.h> 10 #include "trace.h" 11 #include "core.h" 12 13 static DECLARE_RWSEM(cxl_memdev_rwsem); 14 15 /* 16 * An entire PCI topology full of devices should be enough for any 17 * config 18 */ 19 #define CXL_MEM_MAX_DEVS 65536 20 21 static int cxl_mem_major; 22 static DEFINE_IDA(cxl_memdev_ida); 23 24 static void cxl_memdev_release(struct device *dev) 25 { 26 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 27 28 ida_free(&cxl_memdev_ida, cxlmd->id); 29 kfree(cxlmd); 30 } 31 32 static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid, 33 kgid_t *gid) 34 { 35 return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); 36 } 37 38 static ssize_t firmware_version_show(struct device *dev, 39 struct device_attribute *attr, char *buf) 40 { 41 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 42 struct cxl_dev_state *cxlds = cxlmd->cxlds; 43 44 return sysfs_emit(buf, "%.16s\n", cxlds->firmware_version); 45 } 46 static DEVICE_ATTR_RO(firmware_version); 47 48 static ssize_t payload_max_show(struct device *dev, 49 struct device_attribute *attr, char *buf) 50 { 51 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 52 struct cxl_dev_state *cxlds = cxlmd->cxlds; 53 54 return sysfs_emit(buf, "%zu\n", cxlds->payload_size); 55 } 56 static DEVICE_ATTR_RO(payload_max); 57 58 static ssize_t label_storage_size_show(struct device *dev, 59 struct device_attribute *attr, char *buf) 60 { 61 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 62 struct cxl_dev_state *cxlds = cxlmd->cxlds; 63 64 return sysfs_emit(buf, "%zu\n", cxlds->lsa_size); 65 } 66 static DEVICE_ATTR_RO(label_storage_size); 67 68 static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, 69 char *buf) 70 { 71 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 72 struct cxl_dev_state *cxlds = cxlmd->cxlds; 73 unsigned long long len = resource_size(&cxlds->ram_res); 74 75 return sysfs_emit(buf, "%#llx\n", len); 76 } 77 78 static struct device_attribute dev_attr_ram_size = 79 __ATTR(size, 0444, ram_size_show, NULL); 80 81 static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, 82 char *buf) 83 { 84 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 85 struct cxl_dev_state *cxlds = cxlmd->cxlds; 86 unsigned long long len = resource_size(&cxlds->pmem_res); 87 88 return sysfs_emit(buf, "%#llx\n", len); 89 } 90 91 static struct device_attribute dev_attr_pmem_size = 92 __ATTR(size, 0444, pmem_size_show, NULL); 93 94 static ssize_t serial_show(struct device *dev, struct device_attribute *attr, 95 char *buf) 96 { 97 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 98 struct cxl_dev_state *cxlds = cxlmd->cxlds; 99 100 return sysfs_emit(buf, "%#llx\n", cxlds->serial); 101 } 102 static DEVICE_ATTR_RO(serial); 103 104 static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, 105 char *buf) 106 { 107 return sprintf(buf, "%d\n", dev_to_node(dev)); 108 } 109 static DEVICE_ATTR_RO(numa_node); 110 111 static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd) 112 { 113 struct cxl_dev_state *cxlds = cxlmd->cxlds; 114 u64 offset, length; 115 int rc = 0; 116 117 /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */ 118 if (resource_size(&cxlds->pmem_res)) { 119 offset = cxlds->pmem_res.start; 120 length = resource_size(&cxlds->pmem_res); 121 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 122 if (rc) 123 return rc; 124 } 125 if (resource_size(&cxlds->ram_res)) { 126 offset = cxlds->ram_res.start; 127 length = resource_size(&cxlds->ram_res); 128 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 129 /* 130 * Invalid Physical Address is not an error for 131 * volatile addresses. Device support is optional. 132 */ 133 if (rc == -EFAULT) 134 rc = 0; 135 } 136 return rc; 137 } 138 139 int cxl_trigger_poison_list(struct cxl_memdev *cxlmd) 140 { 141 struct cxl_port *port; 142 int rc; 143 144 port = dev_get_drvdata(&cxlmd->dev); 145 if (!port || !is_cxl_endpoint(port)) 146 return -EINVAL; 147 148 rc = down_read_interruptible(&cxl_dpa_rwsem); 149 if (rc) 150 return rc; 151 152 if (port->commit_end == -1) { 153 /* No regions mapped to this memdev */ 154 rc = cxl_get_poison_by_memdev(cxlmd); 155 } else { 156 /* Regions mapped, collect poison by endpoint */ 157 rc = cxl_get_poison_by_endpoint(port); 158 } 159 up_read(&cxl_dpa_rwsem); 160 161 return rc; 162 } 163 EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, CXL); 164 165 struct cxl_dpa_to_region_context { 166 struct cxl_region *cxlr; 167 u64 dpa; 168 }; 169 170 static int __cxl_dpa_to_region(struct device *dev, void *arg) 171 { 172 struct cxl_dpa_to_region_context *ctx = arg; 173 struct cxl_endpoint_decoder *cxled; 174 u64 dpa = ctx->dpa; 175 176 if (!is_endpoint_decoder(dev)) 177 return 0; 178 179 cxled = to_cxl_endpoint_decoder(dev); 180 if (!cxled->dpa_res || !resource_size(cxled->dpa_res)) 181 return 0; 182 183 if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start) 184 return 0; 185 186 dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa, 187 dev_name(&cxled->cxld.region->dev)); 188 189 ctx->cxlr = cxled->cxld.region; 190 191 return 1; 192 } 193 194 static struct cxl_region *cxl_dpa_to_region(struct cxl_memdev *cxlmd, u64 dpa) 195 { 196 struct cxl_dpa_to_region_context ctx; 197 struct cxl_port *port; 198 199 ctx = (struct cxl_dpa_to_region_context) { 200 .dpa = dpa, 201 }; 202 port = dev_get_drvdata(&cxlmd->dev); 203 if (port && is_cxl_endpoint(port) && port->commit_end != -1) 204 device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region); 205 206 return ctx.cxlr; 207 } 208 209 static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa) 210 { 211 struct cxl_dev_state *cxlds = cxlmd->cxlds; 212 213 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 214 return 0; 215 216 if (!resource_size(&cxlds->dpa_res)) { 217 dev_dbg(cxlds->dev, "device has no dpa resource\n"); 218 return -EINVAL; 219 } 220 if (dpa < cxlds->dpa_res.start || dpa > cxlds->dpa_res.end) { 221 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n", 222 dpa, &cxlds->dpa_res); 223 return -EINVAL; 224 } 225 if (!IS_ALIGNED(dpa, 64)) { 226 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa); 227 return -EINVAL; 228 } 229 230 return 0; 231 } 232 233 int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa) 234 { 235 struct cxl_dev_state *cxlds = cxlmd->cxlds; 236 struct cxl_mbox_inject_poison inject; 237 struct cxl_poison_record record; 238 struct cxl_mbox_cmd mbox_cmd; 239 struct cxl_region *cxlr; 240 int rc; 241 242 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 243 return 0; 244 245 rc = down_read_interruptible(&cxl_dpa_rwsem); 246 if (rc) 247 return rc; 248 249 rc = cxl_validate_poison_dpa(cxlmd, dpa); 250 if (rc) 251 goto out; 252 253 inject.address = cpu_to_le64(dpa); 254 mbox_cmd = (struct cxl_mbox_cmd) { 255 .opcode = CXL_MBOX_OP_INJECT_POISON, 256 .size_in = sizeof(inject), 257 .payload_in = &inject, 258 }; 259 rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); 260 if (rc) 261 goto out; 262 263 cxlr = cxl_dpa_to_region(cxlmd, dpa); 264 if (cxlr) 265 dev_warn_once(cxlds->dev, 266 "poison inject dpa:%#llx region: %s\n", dpa, 267 dev_name(&cxlr->dev)); 268 269 record = (struct cxl_poison_record) { 270 .address = cpu_to_le64(dpa), 271 .length = cpu_to_le32(1), 272 }; 273 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT); 274 out: 275 up_read(&cxl_dpa_rwsem); 276 277 return rc; 278 } 279 EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, CXL); 280 281 int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa) 282 { 283 struct cxl_dev_state *cxlds = cxlmd->cxlds; 284 struct cxl_mbox_clear_poison clear; 285 struct cxl_poison_record record; 286 struct cxl_mbox_cmd mbox_cmd; 287 struct cxl_region *cxlr; 288 int rc; 289 290 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 291 return 0; 292 293 rc = down_read_interruptible(&cxl_dpa_rwsem); 294 if (rc) 295 return rc; 296 297 rc = cxl_validate_poison_dpa(cxlmd, dpa); 298 if (rc) 299 goto out; 300 301 /* 302 * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command 303 * is defined to accept 64 bytes of write-data, along with the 304 * address to clear. This driver uses zeroes as write-data. 305 */ 306 clear = (struct cxl_mbox_clear_poison) { 307 .address = cpu_to_le64(dpa) 308 }; 309 310 mbox_cmd = (struct cxl_mbox_cmd) { 311 .opcode = CXL_MBOX_OP_CLEAR_POISON, 312 .size_in = sizeof(clear), 313 .payload_in = &clear, 314 }; 315 316 rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); 317 if (rc) 318 goto out; 319 320 cxlr = cxl_dpa_to_region(cxlmd, dpa); 321 if (cxlr) 322 dev_warn_once(cxlds->dev, "poison clear dpa:%#llx region: %s\n", 323 dpa, dev_name(&cxlr->dev)); 324 325 record = (struct cxl_poison_record) { 326 .address = cpu_to_le64(dpa), 327 .length = cpu_to_le32(1), 328 }; 329 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR); 330 out: 331 up_read(&cxl_dpa_rwsem); 332 333 return rc; 334 } 335 EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, CXL); 336 337 static struct attribute *cxl_memdev_attributes[] = { 338 &dev_attr_serial.attr, 339 &dev_attr_firmware_version.attr, 340 &dev_attr_payload_max.attr, 341 &dev_attr_label_storage_size.attr, 342 &dev_attr_numa_node.attr, 343 NULL, 344 }; 345 346 static struct attribute *cxl_memdev_pmem_attributes[] = { 347 &dev_attr_pmem_size.attr, 348 NULL, 349 }; 350 351 static struct attribute *cxl_memdev_ram_attributes[] = { 352 &dev_attr_ram_size.attr, 353 NULL, 354 }; 355 356 static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a, 357 int n) 358 { 359 if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr) 360 return 0; 361 return a->mode; 362 } 363 364 static struct attribute_group cxl_memdev_attribute_group = { 365 .attrs = cxl_memdev_attributes, 366 .is_visible = cxl_memdev_visible, 367 }; 368 369 static struct attribute_group cxl_memdev_ram_attribute_group = { 370 .name = "ram", 371 .attrs = cxl_memdev_ram_attributes, 372 }; 373 374 static struct attribute_group cxl_memdev_pmem_attribute_group = { 375 .name = "pmem", 376 .attrs = cxl_memdev_pmem_attributes, 377 }; 378 379 static const struct attribute_group *cxl_memdev_attribute_groups[] = { 380 &cxl_memdev_attribute_group, 381 &cxl_memdev_ram_attribute_group, 382 &cxl_memdev_pmem_attribute_group, 383 NULL, 384 }; 385 386 static const struct device_type cxl_memdev_type = { 387 .name = "cxl_memdev", 388 .release = cxl_memdev_release, 389 .devnode = cxl_memdev_devnode, 390 .groups = cxl_memdev_attribute_groups, 391 }; 392 393 bool is_cxl_memdev(const struct device *dev) 394 { 395 return dev->type == &cxl_memdev_type; 396 } 397 EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, CXL); 398 399 /** 400 * set_exclusive_cxl_commands() - atomically disable user cxl commands 401 * @cxlds: The device state to operate on 402 * @cmds: bitmap of commands to mark exclusive 403 * 404 * Grab the cxl_memdev_rwsem in write mode to flush in-flight 405 * invocations of the ioctl path and then disable future execution of 406 * commands with the command ids set in @cmds. 407 */ 408 void set_exclusive_cxl_commands(struct cxl_dev_state *cxlds, unsigned long *cmds) 409 { 410 down_write(&cxl_memdev_rwsem); 411 bitmap_or(cxlds->exclusive_cmds, cxlds->exclusive_cmds, cmds, 412 CXL_MEM_COMMAND_ID_MAX); 413 up_write(&cxl_memdev_rwsem); 414 } 415 EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, CXL); 416 417 /** 418 * clear_exclusive_cxl_commands() - atomically enable user cxl commands 419 * @cxlds: The device state to modify 420 * @cmds: bitmap of commands to mark available for userspace 421 */ 422 void clear_exclusive_cxl_commands(struct cxl_dev_state *cxlds, unsigned long *cmds) 423 { 424 down_write(&cxl_memdev_rwsem); 425 bitmap_andnot(cxlds->exclusive_cmds, cxlds->exclusive_cmds, cmds, 426 CXL_MEM_COMMAND_ID_MAX); 427 up_write(&cxl_memdev_rwsem); 428 } 429 EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, CXL); 430 431 static void cxl_memdev_shutdown(struct device *dev) 432 { 433 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 434 435 down_write(&cxl_memdev_rwsem); 436 cxlmd->cxlds = NULL; 437 up_write(&cxl_memdev_rwsem); 438 } 439 440 static void cxl_memdev_unregister(void *_cxlmd) 441 { 442 struct cxl_memdev *cxlmd = _cxlmd; 443 struct device *dev = &cxlmd->dev; 444 445 cxl_memdev_shutdown(dev); 446 cdev_device_del(&cxlmd->cdev, dev); 447 put_device(dev); 448 } 449 450 static void detach_memdev(struct work_struct *work) 451 { 452 struct cxl_memdev *cxlmd; 453 454 cxlmd = container_of(work, typeof(*cxlmd), detach_work); 455 device_release_driver(&cxlmd->dev); 456 put_device(&cxlmd->dev); 457 } 458 459 static struct lock_class_key cxl_memdev_key; 460 461 static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds, 462 const struct file_operations *fops) 463 { 464 struct cxl_memdev *cxlmd; 465 struct device *dev; 466 struct cdev *cdev; 467 int rc; 468 469 cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL); 470 if (!cxlmd) 471 return ERR_PTR(-ENOMEM); 472 473 rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL); 474 if (rc < 0) 475 goto err; 476 cxlmd->id = rc; 477 cxlmd->depth = -1; 478 479 dev = &cxlmd->dev; 480 device_initialize(dev); 481 lockdep_set_class(&dev->mutex, &cxl_memdev_key); 482 dev->parent = cxlds->dev; 483 dev->bus = &cxl_bus_type; 484 dev->devt = MKDEV(cxl_mem_major, cxlmd->id); 485 dev->type = &cxl_memdev_type; 486 device_set_pm_not_required(dev); 487 INIT_WORK(&cxlmd->detach_work, detach_memdev); 488 489 cdev = &cxlmd->cdev; 490 cdev_init(cdev, fops); 491 return cxlmd; 492 493 err: 494 kfree(cxlmd); 495 return ERR_PTR(rc); 496 } 497 498 static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, 499 unsigned long arg) 500 { 501 switch (cmd) { 502 case CXL_MEM_QUERY_COMMANDS: 503 return cxl_query_cmd(cxlmd, (void __user *)arg); 504 case CXL_MEM_SEND_COMMAND: 505 return cxl_send_cmd(cxlmd, (void __user *)arg); 506 default: 507 return -ENOTTY; 508 } 509 } 510 511 static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, 512 unsigned long arg) 513 { 514 struct cxl_memdev *cxlmd = file->private_data; 515 int rc = -ENXIO; 516 517 down_read(&cxl_memdev_rwsem); 518 if (cxlmd->cxlds) 519 rc = __cxl_memdev_ioctl(cxlmd, cmd, arg); 520 up_read(&cxl_memdev_rwsem); 521 522 return rc; 523 } 524 525 static int cxl_memdev_open(struct inode *inode, struct file *file) 526 { 527 struct cxl_memdev *cxlmd = 528 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 529 530 get_device(&cxlmd->dev); 531 file->private_data = cxlmd; 532 533 return 0; 534 } 535 536 static int cxl_memdev_release_file(struct inode *inode, struct file *file) 537 { 538 struct cxl_memdev *cxlmd = 539 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 540 541 put_device(&cxlmd->dev); 542 543 return 0; 544 } 545 546 /** 547 * cxl_mem_get_fw_info - Get Firmware info 548 * @cxlds: The device data for the operation 549 * 550 * Retrieve firmware info for the device specified. 551 * 552 * Return: 0 if no error: or the result of the mailbox command. 553 * 554 * See CXL-3.0 8.2.9.3.1 Get FW Info 555 */ 556 static int cxl_mem_get_fw_info(struct cxl_dev_state *cxlds) 557 { 558 struct cxl_mbox_get_fw_info info; 559 struct cxl_mbox_cmd mbox_cmd; 560 int rc; 561 562 mbox_cmd = (struct cxl_mbox_cmd) { 563 .opcode = CXL_MBOX_OP_GET_FW_INFO, 564 .size_out = sizeof(info), 565 .payload_out = &info, 566 }; 567 568 rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); 569 if (rc < 0) 570 return rc; 571 572 cxlds->fw.num_slots = info.num_slots; 573 cxlds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK, 574 info.slot_info); 575 576 return 0; 577 } 578 579 /** 580 * cxl_mem_activate_fw - Activate Firmware 581 * @cxlds: The device data for the operation 582 * @slot: slot number to activate 583 * 584 * Activate firmware in a given slot for the device specified. 585 * 586 * Return: 0 if no error: or the result of the mailbox command. 587 * 588 * See CXL-3.0 8.2.9.3.3 Activate FW 589 */ 590 static int cxl_mem_activate_fw(struct cxl_dev_state *cxlds, int slot) 591 { 592 struct cxl_mbox_activate_fw activate; 593 struct cxl_mbox_cmd mbox_cmd; 594 595 if (slot == 0 || slot > cxlds->fw.num_slots) 596 return -EINVAL; 597 598 mbox_cmd = (struct cxl_mbox_cmd) { 599 .opcode = CXL_MBOX_OP_ACTIVATE_FW, 600 .size_in = sizeof(activate), 601 .payload_in = &activate, 602 }; 603 604 /* Only offline activation supported for now */ 605 activate.action = CXL_FW_ACTIVATE_OFFLINE; 606 activate.slot = slot; 607 608 return cxl_internal_send_cmd(cxlds, &mbox_cmd); 609 } 610 611 /** 612 * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer 613 * @cxlds: The device data for the operation 614 * 615 * Abort an in-progress firmware transfer for the device specified. 616 * 617 * Return: 0 if no error: or the result of the mailbox command. 618 * 619 * See CXL-3.0 8.2.9.3.2 Transfer FW 620 */ 621 static int cxl_mem_abort_fw_xfer(struct cxl_dev_state *cxlds) 622 { 623 struct cxl_mbox_transfer_fw *transfer; 624 struct cxl_mbox_cmd mbox_cmd; 625 int rc; 626 627 transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL); 628 if (!transfer) 629 return -ENOMEM; 630 631 /* Set a 1s poll interval and a total wait time of 30s */ 632 mbox_cmd = (struct cxl_mbox_cmd) { 633 .opcode = CXL_MBOX_OP_TRANSFER_FW, 634 .size_in = sizeof(*transfer), 635 .payload_in = transfer, 636 .poll_interval_ms = 1000, 637 .poll_count = 30, 638 }; 639 640 transfer->action = CXL_FW_TRANSFER_ACTION_ABORT; 641 642 rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); 643 kfree(transfer); 644 return rc; 645 } 646 647 static void cxl_fw_cleanup(struct fw_upload *fwl) 648 { 649 struct cxl_dev_state *cxlds = fwl->dd_handle; 650 651 cxlds->fw.next_slot = 0; 652 } 653 654 static int cxl_fw_do_cancel(struct fw_upload *fwl) 655 { 656 struct cxl_dev_state *cxlds = fwl->dd_handle; 657 struct cxl_memdev *cxlmd = cxlds->cxlmd; 658 int rc; 659 660 rc = cxl_mem_abort_fw_xfer(cxlds); 661 if (rc < 0) 662 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc); 663 664 return FW_UPLOAD_ERR_CANCELED; 665 } 666 667 static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data, 668 u32 size) 669 { 670 struct cxl_dev_state *cxlds = fwl->dd_handle; 671 struct cxl_mbox_transfer_fw *transfer; 672 673 if (!size) 674 return FW_UPLOAD_ERR_INVALID_SIZE; 675 676 cxlds->fw.oneshot = struct_size(transfer, data, size) < 677 cxlds->payload_size; 678 679 if (cxl_mem_get_fw_info(cxlds)) 680 return FW_UPLOAD_ERR_HW_ERROR; 681 682 /* 683 * So far no state has been changed, hence no other cleanup is 684 * necessary. Simply return the cancelled status. 685 */ 686 if (test_and_clear_bit(CXL_FW_CANCEL, cxlds->fw.state)) 687 return FW_UPLOAD_ERR_CANCELED; 688 689 return FW_UPLOAD_ERR_NONE; 690 } 691 692 static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data, 693 u32 offset, u32 size, u32 *written) 694 { 695 struct cxl_dev_state *cxlds = fwl->dd_handle; 696 struct cxl_memdev *cxlmd = cxlds->cxlmd; 697 struct cxl_mbox_transfer_fw *transfer; 698 struct cxl_mbox_cmd mbox_cmd; 699 u32 cur_size, remaining; 700 size_t size_in; 701 int rc; 702 703 *written = 0; 704 705 /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */ 706 if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) { 707 dev_err(&cxlmd->dev, 708 "misaligned offset for FW transfer slice (%u)\n", 709 offset); 710 return FW_UPLOAD_ERR_RW_ERROR; 711 } 712 713 /* 714 * Pick transfer size based on cxlds->payload_size 715 * @size must bw 128-byte aligned, ->payload_size is a power of 2 716 * starting at 256 bytes, and sizeof(*transfer) is 128. 717 * These constraints imply that @cur_size will always be 128b aligned. 718 */ 719 cur_size = min_t(size_t, size, cxlds->payload_size - sizeof(*transfer)); 720 721 remaining = size - cur_size; 722 size_in = struct_size(transfer, data, cur_size); 723 724 if (test_and_clear_bit(CXL_FW_CANCEL, cxlds->fw.state)) 725 return cxl_fw_do_cancel(fwl); 726 727 /* 728 * Slot numbers are 1-indexed 729 * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1') 730 * Check for rollover using modulo, and 1-index it by adding 1 731 */ 732 cxlds->fw.next_slot = (cxlds->fw.cur_slot % cxlds->fw.num_slots) + 1; 733 734 /* Do the transfer via mailbox cmd */ 735 transfer = kzalloc(size_in, GFP_KERNEL); 736 if (!transfer) 737 return FW_UPLOAD_ERR_RW_ERROR; 738 739 transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT); 740 memcpy(transfer->data, data + offset, cur_size); 741 if (cxlds->fw.oneshot) { 742 transfer->action = CXL_FW_TRANSFER_ACTION_FULL; 743 transfer->slot = cxlds->fw.next_slot; 744 } else { 745 if (offset == 0) { 746 transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE; 747 } else if (remaining == 0) { 748 transfer->action = CXL_FW_TRANSFER_ACTION_END; 749 transfer->slot = cxlds->fw.next_slot; 750 } else { 751 transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE; 752 } 753 } 754 755 mbox_cmd = (struct cxl_mbox_cmd) { 756 .opcode = CXL_MBOX_OP_TRANSFER_FW, 757 .size_in = size_in, 758 .payload_in = transfer, 759 .poll_interval_ms = 1000, 760 .poll_count = 30, 761 }; 762 763 rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); 764 if (rc < 0) { 765 rc = FW_UPLOAD_ERR_RW_ERROR; 766 goto out_free; 767 } 768 769 *written = cur_size; 770 771 /* Activate FW if oneshot or if the last slice was written */ 772 if (cxlds->fw.oneshot || remaining == 0) { 773 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n", 774 cxlds->fw.next_slot); 775 rc = cxl_mem_activate_fw(cxlds, cxlds->fw.next_slot); 776 if (rc < 0) { 777 dev_err(&cxlmd->dev, "Error activating firmware: %d\n", 778 rc); 779 rc = FW_UPLOAD_ERR_HW_ERROR; 780 goto out_free; 781 } 782 } 783 784 rc = FW_UPLOAD_ERR_NONE; 785 786 out_free: 787 kfree(transfer); 788 return rc; 789 } 790 791 static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl) 792 { 793 struct cxl_dev_state *cxlds = fwl->dd_handle; 794 795 /* 796 * cxl_internal_send_cmd() handles background operations synchronously. 797 * No need to wait for completions here - any errors would've been 798 * reported and handled during the ->write() call(s). 799 * Just check if a cancel request was received, and return success. 800 */ 801 if (test_and_clear_bit(CXL_FW_CANCEL, cxlds->fw.state)) 802 return cxl_fw_do_cancel(fwl); 803 804 return FW_UPLOAD_ERR_NONE; 805 } 806 807 static void cxl_fw_cancel(struct fw_upload *fwl) 808 { 809 struct cxl_dev_state *cxlds = fwl->dd_handle; 810 811 set_bit(CXL_FW_CANCEL, cxlds->fw.state); 812 } 813 814 static const struct fw_upload_ops cxl_memdev_fw_ops = { 815 .prepare = cxl_fw_prepare, 816 .write = cxl_fw_write, 817 .poll_complete = cxl_fw_poll_complete, 818 .cancel = cxl_fw_cancel, 819 .cleanup = cxl_fw_cleanup, 820 }; 821 822 static void devm_cxl_remove_fw_upload(void *fwl) 823 { 824 firmware_upload_unregister(fwl); 825 } 826 827 int cxl_memdev_setup_fw_upload(struct cxl_dev_state *cxlds) 828 { 829 struct device *dev = &cxlds->cxlmd->dev; 830 struct fw_upload *fwl; 831 int rc; 832 833 if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, cxlds->enabled_cmds)) 834 return 0; 835 836 fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev), 837 &cxl_memdev_fw_ops, cxlds); 838 if (IS_ERR(fwl)) 839 return dev_err_probe(dev, PTR_ERR(fwl), 840 "Failed to register firmware loader\n"); 841 842 rc = devm_add_action_or_reset(cxlds->dev, devm_cxl_remove_fw_upload, 843 fwl); 844 if (rc) 845 dev_err(dev, 846 "Failed to add firmware loader remove action: %d\n", 847 rc); 848 849 return rc; 850 } 851 EXPORT_SYMBOL_NS_GPL(cxl_memdev_setup_fw_upload, CXL); 852 853 static const struct file_operations cxl_memdev_fops = { 854 .owner = THIS_MODULE, 855 .unlocked_ioctl = cxl_memdev_ioctl, 856 .open = cxl_memdev_open, 857 .release = cxl_memdev_release_file, 858 .compat_ioctl = compat_ptr_ioctl, 859 .llseek = noop_llseek, 860 }; 861 862 struct cxl_memdev *devm_cxl_add_memdev(struct cxl_dev_state *cxlds) 863 { 864 struct cxl_memdev *cxlmd; 865 struct device *dev; 866 struct cdev *cdev; 867 int rc; 868 869 cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops); 870 if (IS_ERR(cxlmd)) 871 return cxlmd; 872 873 dev = &cxlmd->dev; 874 rc = dev_set_name(dev, "mem%d", cxlmd->id); 875 if (rc) 876 goto err; 877 878 /* 879 * Activate ioctl operations, no cxl_memdev_rwsem manipulation 880 * needed as this is ordered with cdev_add() publishing the device. 881 */ 882 cxlmd->cxlds = cxlds; 883 cxlds->cxlmd = cxlmd; 884 885 cdev = &cxlmd->cdev; 886 rc = cdev_device_add(cdev, dev); 887 if (rc) 888 goto err; 889 890 rc = devm_add_action_or_reset(cxlds->dev, cxl_memdev_unregister, cxlmd); 891 if (rc) 892 return ERR_PTR(rc); 893 return cxlmd; 894 895 err: 896 /* 897 * The cdev was briefly live, shutdown any ioctl operations that 898 * saw that state. 899 */ 900 cxl_memdev_shutdown(dev); 901 put_device(dev); 902 return ERR_PTR(rc); 903 } 904 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL); 905 906 __init int cxl_memdev_init(void) 907 { 908 dev_t devt; 909 int rc; 910 911 rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); 912 if (rc) 913 return rc; 914 915 cxl_mem_major = MAJOR(devt); 916 917 return 0; 918 } 919 920 void cxl_memdev_exit(void) 921 { 922 unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS); 923 } 924