1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */ 3 #include <linux/memregion.h> 4 #include <linux/genalloc.h> 5 #include <linux/device.h> 6 #include <linux/module.h> 7 #include <linux/slab.h> 8 #include <linux/uuid.h> 9 #include <linux/idr.h> 10 #include <cxlmem.h> 11 #include <cxl.h> 12 #include "core.h" 13 14 /** 15 * DOC: cxl core region 16 * 17 * CXL Regions represent mapped memory capacity in system physical address 18 * space. Whereas the CXL Root Decoders identify the bounds of potential CXL 19 * Memory ranges, Regions represent the active mapped capacity by the HDM 20 * Decoder Capability structures throughout the Host Bridges, Switches, and 21 * Endpoints in the topology. 22 * 23 * Region configuration has ordering constraints. UUID may be set at any time 24 * but is only visible for persistent regions. 25 * 1. Interleave granularity 26 * 2. Interleave size 27 * 3. Decoder targets 28 */ 29 30 /* 31 * All changes to the interleave configuration occur with this lock held 32 * for write. 33 */ 34 static DECLARE_RWSEM(cxl_region_rwsem); 35 36 static struct cxl_region *to_cxl_region(struct device *dev); 37 38 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr, 39 char *buf) 40 { 41 struct cxl_region *cxlr = to_cxl_region(dev); 42 struct cxl_region_params *p = &cxlr->params; 43 ssize_t rc; 44 45 rc = down_read_interruptible(&cxl_region_rwsem); 46 if (rc) 47 return rc; 48 rc = sysfs_emit(buf, "%pUb\n", &p->uuid); 49 up_read(&cxl_region_rwsem); 50 51 return rc; 52 } 53 54 static int is_dup(struct device *match, void *data) 55 { 56 struct cxl_region_params *p; 57 struct cxl_region *cxlr; 58 uuid_t *uuid = data; 59 60 if (!is_cxl_region(match)) 61 return 0; 62 63 lockdep_assert_held(&cxl_region_rwsem); 64 cxlr = to_cxl_region(match); 65 p = &cxlr->params; 66 67 if (uuid_equal(&p->uuid, uuid)) { 68 dev_dbg(match, "already has uuid: %pUb\n", uuid); 69 return -EBUSY; 70 } 71 72 return 0; 73 } 74 75 static ssize_t uuid_store(struct device *dev, struct device_attribute *attr, 76 const char *buf, size_t len) 77 { 78 struct cxl_region *cxlr = to_cxl_region(dev); 79 struct cxl_region_params *p = &cxlr->params; 80 uuid_t temp; 81 ssize_t rc; 82 83 if (len != UUID_STRING_LEN + 1) 84 return -EINVAL; 85 86 rc = uuid_parse(buf, &temp); 87 if (rc) 88 return rc; 89 90 if (uuid_is_null(&temp)) 91 return -EINVAL; 92 93 rc = down_write_killable(&cxl_region_rwsem); 94 if (rc) 95 return rc; 96 97 if (uuid_equal(&p->uuid, &temp)) 98 goto out; 99 100 rc = -EBUSY; 101 if (p->state >= CXL_CONFIG_ACTIVE) 102 goto out; 103 104 rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup); 105 if (rc < 0) 106 goto out; 107 108 uuid_copy(&p->uuid, &temp); 109 out: 110 up_write(&cxl_region_rwsem); 111 112 if (rc) 113 return rc; 114 return len; 115 } 116 static DEVICE_ATTR_RW(uuid); 117 118 static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port, 119 struct cxl_region *cxlr) 120 { 121 return xa_load(&port->regions, (unsigned long)cxlr); 122 } 123 124 static int cxl_region_decode_reset(struct cxl_region *cxlr, int count) 125 { 126 struct cxl_region_params *p = &cxlr->params; 127 int i; 128 129 for (i = count - 1; i >= 0; i--) { 130 struct cxl_endpoint_decoder *cxled = p->targets[i]; 131 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 132 struct cxl_port *iter = cxled_to_port(cxled); 133 struct cxl_ep *ep; 134 int rc; 135 136 while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 137 iter = to_cxl_port(iter->dev.parent); 138 139 for (ep = cxl_ep_load(iter, cxlmd); iter; 140 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 141 struct cxl_region_ref *cxl_rr; 142 struct cxl_decoder *cxld; 143 144 cxl_rr = cxl_rr_load(iter, cxlr); 145 cxld = cxl_rr->decoder; 146 rc = cxld->reset(cxld); 147 if (rc) 148 return rc; 149 } 150 151 rc = cxled->cxld.reset(&cxled->cxld); 152 if (rc) 153 return rc; 154 } 155 156 return 0; 157 } 158 159 static int cxl_region_decode_commit(struct cxl_region *cxlr) 160 { 161 struct cxl_region_params *p = &cxlr->params; 162 int i, rc; 163 164 for (i = 0; i < p->nr_targets; i++) { 165 struct cxl_endpoint_decoder *cxled = p->targets[i]; 166 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 167 struct cxl_region_ref *cxl_rr; 168 struct cxl_decoder *cxld; 169 struct cxl_port *iter; 170 struct cxl_ep *ep; 171 172 /* commit bottom up */ 173 for (iter = cxled_to_port(cxled); !is_cxl_root(iter); 174 iter = to_cxl_port(iter->dev.parent)) { 175 cxl_rr = cxl_rr_load(iter, cxlr); 176 cxld = cxl_rr->decoder; 177 rc = cxld->commit(cxld); 178 if (rc) 179 break; 180 } 181 182 /* success, all decoders up to the root are programmed */ 183 if (is_cxl_root(iter)) 184 continue; 185 186 /* programming @iter failed, teardown */ 187 for (ep = cxl_ep_load(iter, cxlmd); ep && iter; 188 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 189 cxl_rr = cxl_rr_load(iter, cxlr); 190 cxld = cxl_rr->decoder; 191 cxld->reset(cxld); 192 } 193 194 cxled->cxld.reset(&cxled->cxld); 195 if (i == 0) 196 return rc; 197 break; 198 } 199 200 if (i >= p->nr_targets) 201 return 0; 202 203 /* undo the targets that were successfully committed */ 204 cxl_region_decode_reset(cxlr, i); 205 return rc; 206 } 207 208 static ssize_t commit_store(struct device *dev, struct device_attribute *attr, 209 const char *buf, size_t len) 210 { 211 struct cxl_region *cxlr = to_cxl_region(dev); 212 struct cxl_region_params *p = &cxlr->params; 213 bool commit; 214 ssize_t rc; 215 216 rc = kstrtobool(buf, &commit); 217 if (rc) 218 return rc; 219 220 rc = down_write_killable(&cxl_region_rwsem); 221 if (rc) 222 return rc; 223 224 /* Already in the requested state? */ 225 if (commit && p->state >= CXL_CONFIG_COMMIT) 226 goto out; 227 if (!commit && p->state < CXL_CONFIG_COMMIT) 228 goto out; 229 230 /* Not ready to commit? */ 231 if (commit && p->state < CXL_CONFIG_ACTIVE) { 232 rc = -ENXIO; 233 goto out; 234 } 235 236 if (commit) 237 rc = cxl_region_decode_commit(cxlr); 238 else { 239 p->state = CXL_CONFIG_RESET_PENDING; 240 up_write(&cxl_region_rwsem); 241 device_release_driver(&cxlr->dev); 242 down_write(&cxl_region_rwsem); 243 244 /* 245 * The lock was dropped, so need to revalidate that the reset is 246 * still pending. 247 */ 248 if (p->state == CXL_CONFIG_RESET_PENDING) 249 rc = cxl_region_decode_reset(cxlr, p->interleave_ways); 250 } 251 252 if (rc) 253 goto out; 254 255 if (commit) 256 p->state = CXL_CONFIG_COMMIT; 257 else if (p->state == CXL_CONFIG_RESET_PENDING) 258 p->state = CXL_CONFIG_ACTIVE; 259 260 out: 261 up_write(&cxl_region_rwsem); 262 263 if (rc) 264 return rc; 265 return len; 266 } 267 268 static ssize_t commit_show(struct device *dev, struct device_attribute *attr, 269 char *buf) 270 { 271 struct cxl_region *cxlr = to_cxl_region(dev); 272 struct cxl_region_params *p = &cxlr->params; 273 ssize_t rc; 274 275 rc = down_read_interruptible(&cxl_region_rwsem); 276 if (rc) 277 return rc; 278 rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT); 279 up_read(&cxl_region_rwsem); 280 281 return rc; 282 } 283 static DEVICE_ATTR_RW(commit); 284 285 static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a, 286 int n) 287 { 288 struct device *dev = kobj_to_dev(kobj); 289 struct cxl_region *cxlr = to_cxl_region(dev); 290 291 if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM) 292 return 0; 293 return a->mode; 294 } 295 296 static ssize_t interleave_ways_show(struct device *dev, 297 struct device_attribute *attr, char *buf) 298 { 299 struct cxl_region *cxlr = to_cxl_region(dev); 300 struct cxl_region_params *p = &cxlr->params; 301 ssize_t rc; 302 303 rc = down_read_interruptible(&cxl_region_rwsem); 304 if (rc) 305 return rc; 306 rc = sysfs_emit(buf, "%d\n", p->interleave_ways); 307 up_read(&cxl_region_rwsem); 308 309 return rc; 310 } 311 312 static const struct attribute_group *get_cxl_region_target_group(void); 313 314 static ssize_t interleave_ways_store(struct device *dev, 315 struct device_attribute *attr, 316 const char *buf, size_t len) 317 { 318 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 319 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 320 struct cxl_region *cxlr = to_cxl_region(dev); 321 struct cxl_region_params *p = &cxlr->params; 322 int rc, val, save; 323 u8 iw; 324 325 rc = kstrtoint(buf, 0, &val); 326 if (rc) 327 return rc; 328 329 rc = ways_to_cxl(val, &iw); 330 if (rc) 331 return rc; 332 333 /* 334 * Even for x3, x9, and x12 interleaves the region interleave must be a 335 * power of 2 multiple of the host bridge interleave. 336 */ 337 if (!is_power_of_2(val / cxld->interleave_ways) || 338 (val % cxld->interleave_ways)) { 339 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val); 340 return -EINVAL; 341 } 342 343 rc = down_write_killable(&cxl_region_rwsem); 344 if (rc) 345 return rc; 346 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 347 rc = -EBUSY; 348 goto out; 349 } 350 351 save = p->interleave_ways; 352 p->interleave_ways = val; 353 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group()); 354 if (rc) 355 p->interleave_ways = save; 356 out: 357 up_write(&cxl_region_rwsem); 358 if (rc) 359 return rc; 360 return len; 361 } 362 static DEVICE_ATTR_RW(interleave_ways); 363 364 static ssize_t interleave_granularity_show(struct device *dev, 365 struct device_attribute *attr, 366 char *buf) 367 { 368 struct cxl_region *cxlr = to_cxl_region(dev); 369 struct cxl_region_params *p = &cxlr->params; 370 ssize_t rc; 371 372 rc = down_read_interruptible(&cxl_region_rwsem); 373 if (rc) 374 return rc; 375 rc = sysfs_emit(buf, "%d\n", p->interleave_granularity); 376 up_read(&cxl_region_rwsem); 377 378 return rc; 379 } 380 381 static ssize_t interleave_granularity_store(struct device *dev, 382 struct device_attribute *attr, 383 const char *buf, size_t len) 384 { 385 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 386 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 387 struct cxl_region *cxlr = to_cxl_region(dev); 388 struct cxl_region_params *p = &cxlr->params; 389 int rc, val; 390 u16 ig; 391 392 rc = kstrtoint(buf, 0, &val); 393 if (rc) 394 return rc; 395 396 rc = granularity_to_cxl(val, &ig); 397 if (rc) 398 return rc; 399 400 /* 401 * Disallow region granularity less than root granularity to 402 * simplify the implementation. Otherwise, region's with a 403 * granularity less than the root interleave result in needing 404 * multiple endpoints to support a single slot in the 405 * interleave. 406 */ 407 if (val < cxld->interleave_granularity) 408 return -EINVAL; 409 410 rc = down_write_killable(&cxl_region_rwsem); 411 if (rc) 412 return rc; 413 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 414 rc = -EBUSY; 415 goto out; 416 } 417 418 p->interleave_granularity = val; 419 out: 420 up_write(&cxl_region_rwsem); 421 if (rc) 422 return rc; 423 return len; 424 } 425 static DEVICE_ATTR_RW(interleave_granularity); 426 427 static ssize_t resource_show(struct device *dev, struct device_attribute *attr, 428 char *buf) 429 { 430 struct cxl_region *cxlr = to_cxl_region(dev); 431 struct cxl_region_params *p = &cxlr->params; 432 u64 resource = -1ULL; 433 ssize_t rc; 434 435 rc = down_read_interruptible(&cxl_region_rwsem); 436 if (rc) 437 return rc; 438 if (p->res) 439 resource = p->res->start; 440 rc = sysfs_emit(buf, "%#llx\n", resource); 441 up_read(&cxl_region_rwsem); 442 443 return rc; 444 } 445 static DEVICE_ATTR_RO(resource); 446 447 static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size) 448 { 449 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 450 struct cxl_region_params *p = &cxlr->params; 451 struct resource *res; 452 u32 remainder = 0; 453 454 lockdep_assert_held_write(&cxl_region_rwsem); 455 456 /* Nothing to do... */ 457 if (p->res && resource_size(res) == size) 458 return 0; 459 460 /* To change size the old size must be freed first */ 461 if (p->res) 462 return -EBUSY; 463 464 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) 465 return -EBUSY; 466 467 /* ways, granularity and uuid (if PMEM) need to be set before HPA */ 468 if (!p->interleave_ways || !p->interleave_granularity || 469 (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid))) 470 return -ENXIO; 471 472 div_u64_rem(size, SZ_256M * p->interleave_ways, &remainder); 473 if (remainder) 474 return -EINVAL; 475 476 res = alloc_free_mem_region(cxlrd->res, size, SZ_256M, 477 dev_name(&cxlr->dev)); 478 if (IS_ERR(res)) { 479 dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n", 480 PTR_ERR(res)); 481 return PTR_ERR(res); 482 } 483 484 p->res = res; 485 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 486 487 return 0; 488 } 489 490 static void cxl_region_iomem_release(struct cxl_region *cxlr) 491 { 492 struct cxl_region_params *p = &cxlr->params; 493 494 if (device_is_registered(&cxlr->dev)) 495 lockdep_assert_held_write(&cxl_region_rwsem); 496 if (p->res) { 497 remove_resource(p->res); 498 kfree(p->res); 499 p->res = NULL; 500 } 501 } 502 503 static int free_hpa(struct cxl_region *cxlr) 504 { 505 struct cxl_region_params *p = &cxlr->params; 506 507 lockdep_assert_held_write(&cxl_region_rwsem); 508 509 if (!p->res) 510 return 0; 511 512 if (p->state >= CXL_CONFIG_ACTIVE) 513 return -EBUSY; 514 515 cxl_region_iomem_release(cxlr); 516 p->state = CXL_CONFIG_IDLE; 517 return 0; 518 } 519 520 static ssize_t size_store(struct device *dev, struct device_attribute *attr, 521 const char *buf, size_t len) 522 { 523 struct cxl_region *cxlr = to_cxl_region(dev); 524 u64 val; 525 int rc; 526 527 rc = kstrtou64(buf, 0, &val); 528 if (rc) 529 return rc; 530 531 rc = down_write_killable(&cxl_region_rwsem); 532 if (rc) 533 return rc; 534 535 if (val) 536 rc = alloc_hpa(cxlr, val); 537 else 538 rc = free_hpa(cxlr); 539 up_write(&cxl_region_rwsem); 540 541 if (rc) 542 return rc; 543 544 return len; 545 } 546 547 static ssize_t size_show(struct device *dev, struct device_attribute *attr, 548 char *buf) 549 { 550 struct cxl_region *cxlr = to_cxl_region(dev); 551 struct cxl_region_params *p = &cxlr->params; 552 u64 size = 0; 553 ssize_t rc; 554 555 rc = down_read_interruptible(&cxl_region_rwsem); 556 if (rc) 557 return rc; 558 if (p->res) 559 size = resource_size(p->res); 560 rc = sysfs_emit(buf, "%#llx\n", size); 561 up_read(&cxl_region_rwsem); 562 563 return rc; 564 } 565 static DEVICE_ATTR_RW(size); 566 567 static struct attribute *cxl_region_attrs[] = { 568 &dev_attr_uuid.attr, 569 &dev_attr_commit.attr, 570 &dev_attr_interleave_ways.attr, 571 &dev_attr_interleave_granularity.attr, 572 &dev_attr_resource.attr, 573 &dev_attr_size.attr, 574 NULL, 575 }; 576 577 static const struct attribute_group cxl_region_group = { 578 .attrs = cxl_region_attrs, 579 .is_visible = cxl_region_visible, 580 }; 581 582 static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos) 583 { 584 struct cxl_region_params *p = &cxlr->params; 585 struct cxl_endpoint_decoder *cxled; 586 int rc; 587 588 rc = down_read_interruptible(&cxl_region_rwsem); 589 if (rc) 590 return rc; 591 592 if (pos >= p->interleave_ways) { 593 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 594 p->interleave_ways); 595 rc = -ENXIO; 596 goto out; 597 } 598 599 cxled = p->targets[pos]; 600 if (!cxled) 601 rc = sysfs_emit(buf, "\n"); 602 else 603 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev)); 604 out: 605 up_read(&cxl_region_rwsem); 606 607 return rc; 608 } 609 610 static int match_free_decoder(struct device *dev, void *data) 611 { 612 struct cxl_decoder *cxld; 613 int *id = data; 614 615 if (!is_switch_decoder(dev)) 616 return 0; 617 618 cxld = to_cxl_decoder(dev); 619 620 /* enforce ordered allocation */ 621 if (cxld->id != *id) 622 return 0; 623 624 if (!cxld->region) 625 return 1; 626 627 (*id)++; 628 629 return 0; 630 } 631 632 static struct cxl_decoder *cxl_region_find_decoder(struct cxl_port *port, 633 struct cxl_region *cxlr) 634 { 635 struct device *dev; 636 int id = 0; 637 638 dev = device_find_child(&port->dev, &id, match_free_decoder); 639 if (!dev) 640 return NULL; 641 /* 642 * This decoder is pinned registered as long as the endpoint decoder is 643 * registered, and endpoint decoder unregistration holds the 644 * cxl_region_rwsem over unregister events, so no need to hold on to 645 * this extra reference. 646 */ 647 put_device(dev); 648 return to_cxl_decoder(dev); 649 } 650 651 static struct cxl_region_ref *alloc_region_ref(struct cxl_port *port, 652 struct cxl_region *cxlr) 653 { 654 struct cxl_region_ref *cxl_rr; 655 int rc; 656 657 cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL); 658 if (!cxl_rr) 659 return NULL; 660 cxl_rr->port = port; 661 cxl_rr->region = cxlr; 662 cxl_rr->nr_targets = 1; 663 xa_init(&cxl_rr->endpoints); 664 665 rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL); 666 if (rc) { 667 dev_dbg(&cxlr->dev, 668 "%s: failed to track region reference: %d\n", 669 dev_name(&port->dev), rc); 670 kfree(cxl_rr); 671 return NULL; 672 } 673 674 return cxl_rr; 675 } 676 677 static void free_region_ref(struct cxl_region_ref *cxl_rr) 678 { 679 struct cxl_port *port = cxl_rr->port; 680 struct cxl_region *cxlr = cxl_rr->region; 681 struct cxl_decoder *cxld = cxl_rr->decoder; 682 683 dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n"); 684 if (cxld->region == cxlr) { 685 cxld->region = NULL; 686 put_device(&cxlr->dev); 687 } 688 689 xa_erase(&port->regions, (unsigned long)cxlr); 690 xa_destroy(&cxl_rr->endpoints); 691 kfree(cxl_rr); 692 } 693 694 static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr, 695 struct cxl_endpoint_decoder *cxled) 696 { 697 int rc; 698 struct cxl_port *port = cxl_rr->port; 699 struct cxl_region *cxlr = cxl_rr->region; 700 struct cxl_decoder *cxld = cxl_rr->decoder; 701 struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled)); 702 703 if (ep) { 704 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep, 705 GFP_KERNEL); 706 if (rc) 707 return rc; 708 } 709 cxl_rr->nr_eps++; 710 711 if (!cxld->region) { 712 cxld->region = cxlr; 713 get_device(&cxlr->dev); 714 } 715 716 return 0; 717 } 718 719 /** 720 * cxl_port_attach_region() - track a region's interest in a port by endpoint 721 * @port: port to add a new region reference 'struct cxl_region_ref' 722 * @cxlr: region to attach to @port 723 * @cxled: endpoint decoder used to create or further pin a region reference 724 * @pos: interleave position of @cxled in @cxlr 725 * 726 * The attach event is an opportunity to validate CXL decode setup 727 * constraints and record metadata needed for programming HDM decoders, 728 * in particular decoder target lists. 729 * 730 * The steps are: 731 * - validate that there are no other regions with a higher HPA already 732 * associated with @port 733 * - establish a region reference if one is not already present 734 * - additionally allocate a decoder instance that will host @cxlr on 735 * @port 736 * - pin the region reference by the endpoint 737 * - account for how many entries in @port's target list are needed to 738 * cover all of the added endpoints. 739 */ 740 static int cxl_port_attach_region(struct cxl_port *port, 741 struct cxl_region *cxlr, 742 struct cxl_endpoint_decoder *cxled, int pos) 743 { 744 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 745 struct cxl_ep *ep = cxl_ep_load(port, cxlmd); 746 struct cxl_region_ref *cxl_rr = NULL, *iter; 747 struct cxl_region_params *p = &cxlr->params; 748 struct cxl_decoder *cxld = NULL; 749 unsigned long index; 750 int rc = -EBUSY; 751 752 lockdep_assert_held_write(&cxl_region_rwsem); 753 754 xa_for_each(&port->regions, index, iter) { 755 struct cxl_region_params *ip = &iter->region->params; 756 757 if (iter->region == cxlr) 758 cxl_rr = iter; 759 if (ip->res->start > p->res->start) { 760 dev_dbg(&cxlr->dev, 761 "%s: HPA order violation %s:%pr vs %pr\n", 762 dev_name(&port->dev), 763 dev_name(&iter->region->dev), ip->res, p->res); 764 return -EBUSY; 765 } 766 } 767 768 if (cxl_rr) { 769 struct cxl_ep *ep_iter; 770 int found = 0; 771 772 cxld = cxl_rr->decoder; 773 xa_for_each(&cxl_rr->endpoints, index, ep_iter) { 774 if (ep_iter == ep) 775 continue; 776 if (ep_iter->next == ep->next) { 777 found++; 778 break; 779 } 780 } 781 782 /* 783 * If this is a new target or if this port is direct connected 784 * to this endpoint then add to the target count. 785 */ 786 if (!found || !ep->next) 787 cxl_rr->nr_targets++; 788 } else { 789 cxl_rr = alloc_region_ref(port, cxlr); 790 if (!cxl_rr) { 791 dev_dbg(&cxlr->dev, 792 "%s: failed to allocate region reference\n", 793 dev_name(&port->dev)); 794 return -ENOMEM; 795 } 796 } 797 798 if (!cxld) { 799 if (port == cxled_to_port(cxled)) 800 cxld = &cxled->cxld; 801 else 802 cxld = cxl_region_find_decoder(port, cxlr); 803 if (!cxld) { 804 dev_dbg(&cxlr->dev, "%s: no decoder available\n", 805 dev_name(&port->dev)); 806 goto out_erase; 807 } 808 809 if (cxld->region) { 810 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n", 811 dev_name(&port->dev), dev_name(&cxld->dev), 812 dev_name(&cxld->region->dev)); 813 rc = -EBUSY; 814 goto out_erase; 815 } 816 817 cxl_rr->decoder = cxld; 818 } 819 820 rc = cxl_rr_ep_add(cxl_rr, cxled); 821 if (rc) { 822 dev_dbg(&cxlr->dev, 823 "%s: failed to track endpoint %s:%s reference\n", 824 dev_name(&port->dev), dev_name(&cxlmd->dev), 825 dev_name(&cxld->dev)); 826 goto out_erase; 827 } 828 829 dev_dbg(&cxlr->dev, 830 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n", 831 dev_name(port->uport), dev_name(&port->dev), 832 dev_name(&cxld->dev), dev_name(&cxlmd->dev), 833 dev_name(&cxled->cxld.dev), pos, 834 ep ? ep->next ? dev_name(ep->next->uport) : 835 dev_name(&cxlmd->dev) : 836 "none", 837 cxl_rr->nr_eps, cxl_rr->nr_targets); 838 839 return 0; 840 out_erase: 841 if (cxl_rr->nr_eps == 0) 842 free_region_ref(cxl_rr); 843 return rc; 844 } 845 846 static void cxl_port_detach_region(struct cxl_port *port, 847 struct cxl_region *cxlr, 848 struct cxl_endpoint_decoder *cxled) 849 { 850 struct cxl_region_ref *cxl_rr; 851 struct cxl_ep *ep = NULL; 852 853 lockdep_assert_held_write(&cxl_region_rwsem); 854 855 cxl_rr = cxl_rr_load(port, cxlr); 856 if (!cxl_rr) 857 return; 858 859 /* 860 * Endpoint ports do not carry cxl_ep references, and they 861 * never target more than one endpoint by definition 862 */ 863 if (cxl_rr->decoder == &cxled->cxld) 864 cxl_rr->nr_eps--; 865 else 866 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled); 867 if (ep) { 868 struct cxl_ep *ep_iter; 869 unsigned long index; 870 int found = 0; 871 872 cxl_rr->nr_eps--; 873 xa_for_each(&cxl_rr->endpoints, index, ep_iter) { 874 if (ep_iter->next == ep->next) { 875 found++; 876 break; 877 } 878 } 879 if (!found) 880 cxl_rr->nr_targets--; 881 } 882 883 if (cxl_rr->nr_eps == 0) 884 free_region_ref(cxl_rr); 885 } 886 887 static int check_last_peer(struct cxl_endpoint_decoder *cxled, 888 struct cxl_ep *ep, struct cxl_region_ref *cxl_rr, 889 int distance) 890 { 891 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 892 struct cxl_region *cxlr = cxl_rr->region; 893 struct cxl_region_params *p = &cxlr->params; 894 struct cxl_endpoint_decoder *cxled_peer; 895 struct cxl_port *port = cxl_rr->port; 896 struct cxl_memdev *cxlmd_peer; 897 struct cxl_ep *ep_peer; 898 int pos = cxled->pos; 899 900 /* 901 * If this position wants to share a dport with the last endpoint mapped 902 * then that endpoint, at index 'position - distance', must also be 903 * mapped by this dport. 904 */ 905 if (pos < distance) { 906 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n", 907 dev_name(port->uport), dev_name(&port->dev), 908 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 909 return -ENXIO; 910 } 911 cxled_peer = p->targets[pos - distance]; 912 cxlmd_peer = cxled_to_memdev(cxled_peer); 913 ep_peer = cxl_ep_load(port, cxlmd_peer); 914 if (ep->dport != ep_peer->dport) { 915 dev_dbg(&cxlr->dev, 916 "%s:%s: %s:%s pos %d mismatched peer %s:%s\n", 917 dev_name(port->uport), dev_name(&port->dev), 918 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos, 919 dev_name(&cxlmd_peer->dev), 920 dev_name(&cxled_peer->cxld.dev)); 921 return -ENXIO; 922 } 923 924 return 0; 925 } 926 927 static int cxl_port_setup_targets(struct cxl_port *port, 928 struct cxl_region *cxlr, 929 struct cxl_endpoint_decoder *cxled) 930 { 931 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 932 int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos; 933 struct cxl_port *parent_port = to_cxl_port(port->dev.parent); 934 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr); 935 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 936 struct cxl_ep *ep = cxl_ep_load(port, cxlmd); 937 struct cxl_region_params *p = &cxlr->params; 938 struct cxl_decoder *cxld = cxl_rr->decoder; 939 struct cxl_switch_decoder *cxlsd; 940 u16 eig, peig; 941 u8 eiw, peiw; 942 943 /* 944 * While root level decoders support x3, x6, x12, switch level 945 * decoders only support powers of 2 up to x16. 946 */ 947 if (!is_power_of_2(cxl_rr->nr_targets)) { 948 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n", 949 dev_name(port->uport), dev_name(&port->dev), 950 cxl_rr->nr_targets); 951 return -EINVAL; 952 } 953 954 cxlsd = to_cxl_switch_decoder(&cxld->dev); 955 if (cxl_rr->nr_targets_set) { 956 int i, distance; 957 958 distance = p->nr_targets / cxl_rr->nr_targets; 959 for (i = 0; i < cxl_rr->nr_targets_set; i++) 960 if (ep->dport == cxlsd->target[i]) { 961 rc = check_last_peer(cxled, ep, cxl_rr, 962 distance); 963 if (rc) 964 return rc; 965 goto out_target_set; 966 } 967 goto add_target; 968 } 969 970 if (is_cxl_root(parent_port)) { 971 parent_ig = cxlrd->cxlsd.cxld.interleave_granularity; 972 parent_iw = cxlrd->cxlsd.cxld.interleave_ways; 973 /* 974 * For purposes of address bit routing, use power-of-2 math for 975 * switch ports. 976 */ 977 if (!is_power_of_2(parent_iw)) 978 parent_iw /= 3; 979 } else { 980 struct cxl_region_ref *parent_rr; 981 struct cxl_decoder *parent_cxld; 982 983 parent_rr = cxl_rr_load(parent_port, cxlr); 984 parent_cxld = parent_rr->decoder; 985 parent_ig = parent_cxld->interleave_granularity; 986 parent_iw = parent_cxld->interleave_ways; 987 } 988 989 granularity_to_cxl(parent_ig, &peig); 990 ways_to_cxl(parent_iw, &peiw); 991 992 iw = cxl_rr->nr_targets; 993 ways_to_cxl(iw, &eiw); 994 if (cxl_rr->nr_targets > 1) { 995 u32 address_bit = max(peig + peiw, eiw + peig); 996 997 eig = address_bit - eiw + 1; 998 } else { 999 eiw = peiw; 1000 eig = peig; 1001 } 1002 1003 rc = cxl_to_granularity(eig, &ig); 1004 if (rc) { 1005 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n", 1006 dev_name(port->uport), dev_name(&port->dev), 1007 256 << eig); 1008 return rc; 1009 } 1010 1011 cxld->interleave_ways = iw; 1012 cxld->interleave_granularity = ig; 1013 dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport), 1014 dev_name(&port->dev), iw, ig); 1015 add_target: 1016 if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) { 1017 dev_dbg(&cxlr->dev, 1018 "%s:%s: targets full trying to add %s:%s at %d\n", 1019 dev_name(port->uport), dev_name(&port->dev), 1020 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 1021 return -ENXIO; 1022 } 1023 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport; 1024 inc = 1; 1025 out_target_set: 1026 cxl_rr->nr_targets_set += inc; 1027 dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n", 1028 dev_name(port->uport), dev_name(&port->dev), 1029 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport), 1030 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 1031 1032 return 0; 1033 } 1034 1035 static void cxl_port_reset_targets(struct cxl_port *port, 1036 struct cxl_region *cxlr) 1037 { 1038 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr); 1039 1040 /* 1041 * After the last endpoint has been detached the entire cxl_rr may now 1042 * be gone. 1043 */ 1044 if (cxl_rr) 1045 cxl_rr->nr_targets_set = 0; 1046 } 1047 1048 static void cxl_region_teardown_targets(struct cxl_region *cxlr) 1049 { 1050 struct cxl_region_params *p = &cxlr->params; 1051 struct cxl_endpoint_decoder *cxled; 1052 struct cxl_memdev *cxlmd; 1053 struct cxl_port *iter; 1054 struct cxl_ep *ep; 1055 int i; 1056 1057 for (i = 0; i < p->nr_targets; i++) { 1058 cxled = p->targets[i]; 1059 cxlmd = cxled_to_memdev(cxled); 1060 1061 iter = cxled_to_port(cxled); 1062 while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 1063 iter = to_cxl_port(iter->dev.parent); 1064 1065 for (ep = cxl_ep_load(iter, cxlmd); iter; 1066 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) 1067 cxl_port_reset_targets(iter, cxlr); 1068 } 1069 } 1070 1071 static int cxl_region_setup_targets(struct cxl_region *cxlr) 1072 { 1073 struct cxl_region_params *p = &cxlr->params; 1074 struct cxl_endpoint_decoder *cxled; 1075 struct cxl_memdev *cxlmd; 1076 struct cxl_port *iter; 1077 struct cxl_ep *ep; 1078 int i, rc; 1079 1080 for (i = 0; i < p->nr_targets; i++) { 1081 cxled = p->targets[i]; 1082 cxlmd = cxled_to_memdev(cxled); 1083 1084 iter = cxled_to_port(cxled); 1085 while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 1086 iter = to_cxl_port(iter->dev.parent); 1087 1088 /* 1089 * Descend the topology tree programming targets while 1090 * looking for conflicts. 1091 */ 1092 for (ep = cxl_ep_load(iter, cxlmd); iter; 1093 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 1094 rc = cxl_port_setup_targets(iter, cxlr, cxled); 1095 if (rc) { 1096 cxl_region_teardown_targets(cxlr); 1097 return rc; 1098 } 1099 } 1100 } 1101 1102 return 0; 1103 } 1104 1105 static int cxl_region_attach(struct cxl_region *cxlr, 1106 struct cxl_endpoint_decoder *cxled, int pos) 1107 { 1108 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 1109 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1110 struct cxl_port *ep_port, *root_port, *iter; 1111 struct cxl_region_params *p = &cxlr->params; 1112 struct cxl_dport *dport; 1113 int i, rc = -ENXIO; 1114 1115 if (cxled->mode == CXL_DECODER_DEAD) { 1116 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev)); 1117 return -ENODEV; 1118 } 1119 1120 /* all full of members, or interleave config not established? */ 1121 if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) { 1122 dev_dbg(&cxlr->dev, "region already active\n"); 1123 return -EBUSY; 1124 } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) { 1125 dev_dbg(&cxlr->dev, "interleave config missing\n"); 1126 return -ENXIO; 1127 } 1128 1129 if (pos < 0 || pos >= p->interleave_ways) { 1130 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 1131 p->interleave_ways); 1132 return -ENXIO; 1133 } 1134 1135 if (p->targets[pos] == cxled) 1136 return 0; 1137 1138 if (p->targets[pos]) { 1139 struct cxl_endpoint_decoder *cxled_target = p->targets[pos]; 1140 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target); 1141 1142 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n", 1143 pos, dev_name(&cxlmd_target->dev), 1144 dev_name(&cxled_target->cxld.dev)); 1145 return -EBUSY; 1146 } 1147 1148 for (i = 0; i < p->interleave_ways; i++) { 1149 struct cxl_endpoint_decoder *cxled_target; 1150 struct cxl_memdev *cxlmd_target; 1151 1152 cxled_target = p->targets[pos]; 1153 if (!cxled_target) 1154 continue; 1155 1156 cxlmd_target = cxled_to_memdev(cxled_target); 1157 if (cxlmd_target == cxlmd) { 1158 dev_dbg(&cxlr->dev, 1159 "%s already specified at position %d via: %s\n", 1160 dev_name(&cxlmd->dev), pos, 1161 dev_name(&cxled_target->cxld.dev)); 1162 return -EBUSY; 1163 } 1164 } 1165 1166 ep_port = cxled_to_port(cxled); 1167 root_port = cxlrd_to_port(cxlrd); 1168 dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge); 1169 if (!dport) { 1170 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n", 1171 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1172 dev_name(cxlr->dev.parent)); 1173 return -ENXIO; 1174 } 1175 1176 if (cxlrd->calc_hb(cxlrd, pos) != dport) { 1177 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n", 1178 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1179 dev_name(&cxlrd->cxlsd.cxld.dev)); 1180 return -ENXIO; 1181 } 1182 1183 if (cxled->cxld.target_type != cxlr->type) { 1184 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n", 1185 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1186 cxled->cxld.target_type, cxlr->type); 1187 return -ENXIO; 1188 } 1189 1190 if (!cxled->dpa_res) { 1191 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n", 1192 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev)); 1193 return -ENXIO; 1194 } 1195 1196 if (resource_size(cxled->dpa_res) * p->interleave_ways != 1197 resource_size(p->res)) { 1198 dev_dbg(&cxlr->dev, 1199 "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n", 1200 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1201 (u64)resource_size(cxled->dpa_res), p->interleave_ways, 1202 (u64)resource_size(p->res)); 1203 return -EINVAL; 1204 } 1205 1206 for (iter = ep_port; !is_cxl_root(iter); 1207 iter = to_cxl_port(iter->dev.parent)) { 1208 rc = cxl_port_attach_region(iter, cxlr, cxled, pos); 1209 if (rc) 1210 goto err; 1211 } 1212 1213 p->targets[pos] = cxled; 1214 cxled->pos = pos; 1215 p->nr_targets++; 1216 1217 if (p->nr_targets == p->interleave_ways) { 1218 rc = cxl_region_setup_targets(cxlr); 1219 if (rc) 1220 goto err; 1221 p->state = CXL_CONFIG_ACTIVE; 1222 } 1223 1224 return 0; 1225 1226 err: 1227 for (iter = ep_port; !is_cxl_root(iter); 1228 iter = to_cxl_port(iter->dev.parent)) 1229 cxl_port_detach_region(iter, cxlr, cxled); 1230 return rc; 1231 } 1232 1233 static int cxl_region_detach(struct cxl_endpoint_decoder *cxled) 1234 { 1235 struct cxl_port *iter, *ep_port = cxled_to_port(cxled); 1236 struct cxl_region *cxlr = cxled->cxld.region; 1237 struct cxl_region_params *p; 1238 int rc = 0; 1239 1240 lockdep_assert_held_write(&cxl_region_rwsem); 1241 1242 if (!cxlr) 1243 return 0; 1244 1245 p = &cxlr->params; 1246 get_device(&cxlr->dev); 1247 1248 if (p->state > CXL_CONFIG_ACTIVE) { 1249 /* 1250 * TODO: tear down all impacted regions if a device is 1251 * removed out of order 1252 */ 1253 rc = cxl_region_decode_reset(cxlr, p->interleave_ways); 1254 if (rc) 1255 goto out; 1256 p->state = CXL_CONFIG_ACTIVE; 1257 } 1258 1259 for (iter = ep_port; !is_cxl_root(iter); 1260 iter = to_cxl_port(iter->dev.parent)) 1261 cxl_port_detach_region(iter, cxlr, cxled); 1262 1263 if (cxled->pos < 0 || cxled->pos >= p->interleave_ways || 1264 p->targets[cxled->pos] != cxled) { 1265 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1266 1267 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n", 1268 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1269 cxled->pos); 1270 goto out; 1271 } 1272 1273 if (p->state == CXL_CONFIG_ACTIVE) { 1274 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 1275 cxl_region_teardown_targets(cxlr); 1276 } 1277 p->targets[cxled->pos] = NULL; 1278 p->nr_targets--; 1279 1280 /* notify the region driver that one of its targets has departed */ 1281 up_write(&cxl_region_rwsem); 1282 device_release_driver(&cxlr->dev); 1283 down_write(&cxl_region_rwsem); 1284 out: 1285 put_device(&cxlr->dev); 1286 return rc; 1287 } 1288 1289 void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled) 1290 { 1291 down_write(&cxl_region_rwsem); 1292 cxled->mode = CXL_DECODER_DEAD; 1293 cxl_region_detach(cxled); 1294 up_write(&cxl_region_rwsem); 1295 } 1296 1297 static int attach_target(struct cxl_region *cxlr, const char *decoder, int pos) 1298 { 1299 struct device *dev; 1300 int rc; 1301 1302 dev = bus_find_device_by_name(&cxl_bus_type, NULL, decoder); 1303 if (!dev) 1304 return -ENODEV; 1305 1306 if (!is_endpoint_decoder(dev)) { 1307 put_device(dev); 1308 return -EINVAL; 1309 } 1310 1311 rc = down_write_killable(&cxl_region_rwsem); 1312 if (rc) 1313 goto out; 1314 down_read(&cxl_dpa_rwsem); 1315 rc = cxl_region_attach(cxlr, to_cxl_endpoint_decoder(dev), pos); 1316 up_read(&cxl_dpa_rwsem); 1317 up_write(&cxl_region_rwsem); 1318 out: 1319 put_device(dev); 1320 return rc; 1321 } 1322 1323 static int detach_target(struct cxl_region *cxlr, int pos) 1324 { 1325 struct cxl_region_params *p = &cxlr->params; 1326 int rc; 1327 1328 rc = down_write_killable(&cxl_region_rwsem); 1329 if (rc) 1330 return rc; 1331 1332 if (pos >= p->interleave_ways) { 1333 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 1334 p->interleave_ways); 1335 rc = -ENXIO; 1336 goto out; 1337 } 1338 1339 if (!p->targets[pos]) { 1340 rc = 0; 1341 goto out; 1342 } 1343 1344 rc = cxl_region_detach(p->targets[pos]); 1345 out: 1346 up_write(&cxl_region_rwsem); 1347 return rc; 1348 } 1349 1350 static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos, 1351 size_t len) 1352 { 1353 int rc; 1354 1355 if (sysfs_streq(buf, "\n")) 1356 rc = detach_target(cxlr, pos); 1357 else 1358 rc = attach_target(cxlr, buf, pos); 1359 1360 if (rc < 0) 1361 return rc; 1362 return len; 1363 } 1364 1365 #define TARGET_ATTR_RW(n) \ 1366 static ssize_t target##n##_show( \ 1367 struct device *dev, struct device_attribute *attr, char *buf) \ 1368 { \ 1369 return show_targetN(to_cxl_region(dev), buf, (n)); \ 1370 } \ 1371 static ssize_t target##n##_store(struct device *dev, \ 1372 struct device_attribute *attr, \ 1373 const char *buf, size_t len) \ 1374 { \ 1375 return store_targetN(to_cxl_region(dev), buf, (n), len); \ 1376 } \ 1377 static DEVICE_ATTR_RW(target##n) 1378 1379 TARGET_ATTR_RW(0); 1380 TARGET_ATTR_RW(1); 1381 TARGET_ATTR_RW(2); 1382 TARGET_ATTR_RW(3); 1383 TARGET_ATTR_RW(4); 1384 TARGET_ATTR_RW(5); 1385 TARGET_ATTR_RW(6); 1386 TARGET_ATTR_RW(7); 1387 TARGET_ATTR_RW(8); 1388 TARGET_ATTR_RW(9); 1389 TARGET_ATTR_RW(10); 1390 TARGET_ATTR_RW(11); 1391 TARGET_ATTR_RW(12); 1392 TARGET_ATTR_RW(13); 1393 TARGET_ATTR_RW(14); 1394 TARGET_ATTR_RW(15); 1395 1396 static struct attribute *target_attrs[] = { 1397 &dev_attr_target0.attr, 1398 &dev_attr_target1.attr, 1399 &dev_attr_target2.attr, 1400 &dev_attr_target3.attr, 1401 &dev_attr_target4.attr, 1402 &dev_attr_target5.attr, 1403 &dev_attr_target6.attr, 1404 &dev_attr_target7.attr, 1405 &dev_attr_target8.attr, 1406 &dev_attr_target9.attr, 1407 &dev_attr_target10.attr, 1408 &dev_attr_target11.attr, 1409 &dev_attr_target12.attr, 1410 &dev_attr_target13.attr, 1411 &dev_attr_target14.attr, 1412 &dev_attr_target15.attr, 1413 NULL, 1414 }; 1415 1416 static umode_t cxl_region_target_visible(struct kobject *kobj, 1417 struct attribute *a, int n) 1418 { 1419 struct device *dev = kobj_to_dev(kobj); 1420 struct cxl_region *cxlr = to_cxl_region(dev); 1421 struct cxl_region_params *p = &cxlr->params; 1422 1423 if (n < p->interleave_ways) 1424 return a->mode; 1425 return 0; 1426 } 1427 1428 static const struct attribute_group cxl_region_target_group = { 1429 .attrs = target_attrs, 1430 .is_visible = cxl_region_target_visible, 1431 }; 1432 1433 static const struct attribute_group *get_cxl_region_target_group(void) 1434 { 1435 return &cxl_region_target_group; 1436 } 1437 1438 static const struct attribute_group *region_groups[] = { 1439 &cxl_base_attribute_group, 1440 &cxl_region_group, 1441 &cxl_region_target_group, 1442 NULL, 1443 }; 1444 1445 static void cxl_region_release(struct device *dev) 1446 { 1447 struct cxl_region *cxlr = to_cxl_region(dev); 1448 1449 memregion_free(cxlr->id); 1450 kfree(cxlr); 1451 } 1452 1453 const struct device_type cxl_region_type = { 1454 .name = "cxl_region", 1455 .release = cxl_region_release, 1456 .groups = region_groups 1457 }; 1458 1459 bool is_cxl_region(struct device *dev) 1460 { 1461 return dev->type == &cxl_region_type; 1462 } 1463 EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL); 1464 1465 static struct cxl_region *to_cxl_region(struct device *dev) 1466 { 1467 if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type, 1468 "not a cxl_region device\n")) 1469 return NULL; 1470 1471 return container_of(dev, struct cxl_region, dev); 1472 } 1473 1474 static void unregister_region(void *dev) 1475 { 1476 struct cxl_region *cxlr = to_cxl_region(dev); 1477 1478 device_del(dev); 1479 cxl_region_iomem_release(cxlr); 1480 put_device(dev); 1481 } 1482 1483 static struct lock_class_key cxl_region_key; 1484 1485 static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id) 1486 { 1487 struct cxl_region *cxlr; 1488 struct device *dev; 1489 1490 cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL); 1491 if (!cxlr) { 1492 memregion_free(id); 1493 return ERR_PTR(-ENOMEM); 1494 } 1495 1496 dev = &cxlr->dev; 1497 device_initialize(dev); 1498 lockdep_set_class(&dev->mutex, &cxl_region_key); 1499 dev->parent = &cxlrd->cxlsd.cxld.dev; 1500 device_set_pm_not_required(dev); 1501 dev->bus = &cxl_bus_type; 1502 dev->type = &cxl_region_type; 1503 cxlr->id = id; 1504 1505 return cxlr; 1506 } 1507 1508 /** 1509 * devm_cxl_add_region - Adds a region to a decoder 1510 * @cxlrd: root decoder 1511 * @id: memregion id to create, or memregion_free() on failure 1512 * @mode: mode for the endpoint decoders of this region 1513 * @type: select whether this is an expander or accelerator (type-2 or type-3) 1514 * 1515 * This is the second step of region initialization. Regions exist within an 1516 * address space which is mapped by a @cxlrd. 1517 * 1518 * Return: 0 if the region was added to the @cxlrd, else returns negative error 1519 * code. The region will be named "regionZ" where Z is the unique region number. 1520 */ 1521 static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd, 1522 int id, 1523 enum cxl_decoder_mode mode, 1524 enum cxl_decoder_type type) 1525 { 1526 struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent); 1527 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 1528 struct cxl_region_params *p; 1529 struct cxl_region *cxlr; 1530 struct device *dev; 1531 int rc; 1532 1533 cxlr = cxl_region_alloc(cxlrd, id); 1534 if (IS_ERR(cxlr)) 1535 return cxlr; 1536 p = &cxlr->params; 1537 cxlr->mode = mode; 1538 cxlr->type = type; 1539 p->interleave_granularity = cxld->interleave_granularity; 1540 1541 dev = &cxlr->dev; 1542 rc = dev_set_name(dev, "region%d", id); 1543 if (rc) 1544 goto err; 1545 1546 rc = device_add(dev); 1547 if (rc) 1548 goto err; 1549 1550 rc = devm_add_action_or_reset(port->uport, unregister_region, cxlr); 1551 if (rc) 1552 return ERR_PTR(rc); 1553 1554 dev_dbg(port->uport, "%s: created %s\n", 1555 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev)); 1556 return cxlr; 1557 1558 err: 1559 put_device(dev); 1560 return ERR_PTR(rc); 1561 } 1562 1563 static ssize_t create_pmem_region_show(struct device *dev, 1564 struct device_attribute *attr, char *buf) 1565 { 1566 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 1567 1568 return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id)); 1569 } 1570 1571 static ssize_t create_pmem_region_store(struct device *dev, 1572 struct device_attribute *attr, 1573 const char *buf, size_t len) 1574 { 1575 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 1576 struct cxl_region *cxlr; 1577 int id, rc; 1578 1579 rc = sscanf(buf, "region%d\n", &id); 1580 if (rc != 1) 1581 return -EINVAL; 1582 1583 rc = memregion_alloc(GFP_KERNEL); 1584 if (rc < 0) 1585 return rc; 1586 1587 if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) { 1588 memregion_free(rc); 1589 return -EBUSY; 1590 } 1591 1592 cxlr = devm_cxl_add_region(cxlrd, id, CXL_DECODER_PMEM, 1593 CXL_DECODER_EXPANDER); 1594 if (IS_ERR(cxlr)) 1595 return PTR_ERR(cxlr); 1596 1597 return len; 1598 } 1599 DEVICE_ATTR_RW(create_pmem_region); 1600 1601 static ssize_t region_show(struct device *dev, struct device_attribute *attr, 1602 char *buf) 1603 { 1604 struct cxl_decoder *cxld = to_cxl_decoder(dev); 1605 ssize_t rc; 1606 1607 rc = down_read_interruptible(&cxl_region_rwsem); 1608 if (rc) 1609 return rc; 1610 1611 if (cxld->region) 1612 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev)); 1613 else 1614 rc = sysfs_emit(buf, "\n"); 1615 up_read(&cxl_region_rwsem); 1616 1617 return rc; 1618 } 1619 DEVICE_ATTR_RO(region); 1620 1621 static struct cxl_region * 1622 cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name) 1623 { 1624 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 1625 struct device *region_dev; 1626 1627 region_dev = device_find_child_by_name(&cxld->dev, name); 1628 if (!region_dev) 1629 return ERR_PTR(-ENODEV); 1630 1631 return to_cxl_region(region_dev); 1632 } 1633 1634 static ssize_t delete_region_store(struct device *dev, 1635 struct device_attribute *attr, 1636 const char *buf, size_t len) 1637 { 1638 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 1639 struct cxl_port *port = to_cxl_port(dev->parent); 1640 struct cxl_region *cxlr; 1641 1642 cxlr = cxl_find_region_by_name(cxlrd, buf); 1643 if (IS_ERR(cxlr)) 1644 return PTR_ERR(cxlr); 1645 1646 devm_release_action(port->uport, unregister_region, cxlr); 1647 put_device(&cxlr->dev); 1648 1649 return len; 1650 } 1651 DEVICE_ATTR_WO(delete_region); 1652 1653 static void cxl_pmem_region_release(struct device *dev) 1654 { 1655 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev); 1656 int i; 1657 1658 for (i = 0; i < cxlr_pmem->nr_mappings; i++) { 1659 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd; 1660 1661 put_device(&cxlmd->dev); 1662 } 1663 1664 kfree(cxlr_pmem); 1665 } 1666 1667 static const struct attribute_group *cxl_pmem_region_attribute_groups[] = { 1668 &cxl_base_attribute_group, 1669 NULL, 1670 }; 1671 1672 const struct device_type cxl_pmem_region_type = { 1673 .name = "cxl_pmem_region", 1674 .release = cxl_pmem_region_release, 1675 .groups = cxl_pmem_region_attribute_groups, 1676 }; 1677 1678 bool is_cxl_pmem_region(struct device *dev) 1679 { 1680 return dev->type == &cxl_pmem_region_type; 1681 } 1682 EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL); 1683 1684 struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev) 1685 { 1686 if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev), 1687 "not a cxl_pmem_region device\n")) 1688 return NULL; 1689 return container_of(dev, struct cxl_pmem_region, dev); 1690 } 1691 EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL); 1692 1693 static struct lock_class_key cxl_pmem_region_key; 1694 1695 static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr) 1696 { 1697 struct cxl_region_params *p = &cxlr->params; 1698 struct cxl_pmem_region *cxlr_pmem; 1699 struct device *dev; 1700 int i; 1701 1702 down_read(&cxl_region_rwsem); 1703 if (p->state != CXL_CONFIG_COMMIT) { 1704 cxlr_pmem = ERR_PTR(-ENXIO); 1705 goto out; 1706 } 1707 1708 cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets), 1709 GFP_KERNEL); 1710 if (!cxlr_pmem) { 1711 cxlr_pmem = ERR_PTR(-ENOMEM); 1712 goto out; 1713 } 1714 1715 cxlr_pmem->hpa_range.start = p->res->start; 1716 cxlr_pmem->hpa_range.end = p->res->end; 1717 1718 /* Snapshot the region configuration underneath the cxl_region_rwsem */ 1719 cxlr_pmem->nr_mappings = p->nr_targets; 1720 for (i = 0; i < p->nr_targets; i++) { 1721 struct cxl_endpoint_decoder *cxled = p->targets[i]; 1722 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1723 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i]; 1724 1725 m->cxlmd = cxlmd; 1726 get_device(&cxlmd->dev); 1727 m->start = cxled->dpa_res->start; 1728 m->size = resource_size(cxled->dpa_res); 1729 m->position = i; 1730 } 1731 1732 dev = &cxlr_pmem->dev; 1733 cxlr_pmem->cxlr = cxlr; 1734 device_initialize(dev); 1735 lockdep_set_class(&dev->mutex, &cxl_pmem_region_key); 1736 device_set_pm_not_required(dev); 1737 dev->parent = &cxlr->dev; 1738 dev->bus = &cxl_bus_type; 1739 dev->type = &cxl_pmem_region_type; 1740 out: 1741 up_read(&cxl_region_rwsem); 1742 1743 return cxlr_pmem; 1744 } 1745 1746 static void cxlr_pmem_unregister(void *dev) 1747 { 1748 device_unregister(dev); 1749 } 1750 1751 /** 1752 * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge 1753 * @cxlr: parent CXL region for this pmem region bridge device 1754 * 1755 * Return: 0 on success negative error code on failure. 1756 */ 1757 static int devm_cxl_add_pmem_region(struct cxl_region *cxlr) 1758 { 1759 struct cxl_pmem_region *cxlr_pmem; 1760 struct device *dev; 1761 int rc; 1762 1763 cxlr_pmem = cxl_pmem_region_alloc(cxlr); 1764 if (IS_ERR(cxlr_pmem)) 1765 return PTR_ERR(cxlr_pmem); 1766 1767 dev = &cxlr_pmem->dev; 1768 rc = dev_set_name(dev, "pmem_region%d", cxlr->id); 1769 if (rc) 1770 goto err; 1771 1772 rc = device_add(dev); 1773 if (rc) 1774 goto err; 1775 1776 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent), 1777 dev_name(dev)); 1778 1779 return devm_add_action_or_reset(&cxlr->dev, cxlr_pmem_unregister, dev); 1780 1781 err: 1782 put_device(dev); 1783 return rc; 1784 } 1785 1786 static int cxl_region_probe(struct device *dev) 1787 { 1788 struct cxl_region *cxlr = to_cxl_region(dev); 1789 struct cxl_region_params *p = &cxlr->params; 1790 int rc; 1791 1792 rc = down_read_interruptible(&cxl_region_rwsem); 1793 if (rc) { 1794 dev_dbg(&cxlr->dev, "probe interrupted\n"); 1795 return rc; 1796 } 1797 1798 if (p->state < CXL_CONFIG_COMMIT) { 1799 dev_dbg(&cxlr->dev, "config state: %d\n", p->state); 1800 rc = -ENXIO; 1801 } 1802 1803 /* 1804 * From this point on any path that changes the region's state away from 1805 * CXL_CONFIG_COMMIT is also responsible for releasing the driver. 1806 */ 1807 up_read(&cxl_region_rwsem); 1808 1809 switch (cxlr->mode) { 1810 case CXL_DECODER_PMEM: 1811 return devm_cxl_add_pmem_region(cxlr); 1812 default: 1813 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n", 1814 cxlr->mode); 1815 return -ENXIO; 1816 } 1817 } 1818 1819 static struct cxl_driver cxl_region_driver = { 1820 .name = "cxl_region", 1821 .probe = cxl_region_probe, 1822 .id = CXL_DEVICE_REGION, 1823 }; 1824 1825 int cxl_region_init(void) 1826 { 1827 return cxl_driver_register(&cxl_region_driver); 1828 } 1829 1830 void cxl_region_exit(void) 1831 { 1832 cxl_driver_unregister(&cxl_region_driver); 1833 } 1834 1835 MODULE_IMPORT_NS(CXL); 1836 MODULE_ALIAS_CXL(CXL_DEVICE_REGION); 1837