1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */ 3 #include <linux/io-64-nonatomic-hi-lo.h> 4 #include <linux/seq_file.h> 5 #include <linux/device.h> 6 #include <linux/delay.h> 7 8 #include "cxlmem.h" 9 #include "core.h" 10 11 /** 12 * DOC: cxl core hdm 13 * 14 * Compute Express Link Host Managed Device Memory, starting with the 15 * CXL 2.0 specification, is managed by an array of HDM Decoder register 16 * instances per CXL port and per CXL endpoint. Define common helpers 17 * for enumerating these registers and capabilities. 18 */ 19 20 DECLARE_RWSEM(cxl_dpa_rwsem); 21 22 static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, 23 int *target_map) 24 { 25 int rc; 26 27 rc = cxl_decoder_add_locked(cxld, target_map); 28 if (rc) { 29 put_device(&cxld->dev); 30 dev_err(&port->dev, "Failed to add decoder\n"); 31 return rc; 32 } 33 34 rc = cxl_decoder_autoremove(&port->dev, cxld); 35 if (rc) 36 return rc; 37 38 dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev)); 39 40 return 0; 41 } 42 43 /* 44 * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure) 45 * single ported host-bridges need not publish a decoder capability when a 46 * passthrough decode can be assumed, i.e. all transactions that the uport sees 47 * are claimed and passed to the single dport. Disable the range until the first 48 * CXL region is enumerated / activated. 49 */ 50 int devm_cxl_add_passthrough_decoder(struct cxl_port *port) 51 { 52 struct cxl_switch_decoder *cxlsd; 53 struct cxl_dport *dport = NULL; 54 int single_port_map[1]; 55 unsigned long index; 56 57 cxlsd = cxl_switch_decoder_alloc(port, 1); 58 if (IS_ERR(cxlsd)) 59 return PTR_ERR(cxlsd); 60 61 device_lock_assert(&port->dev); 62 63 xa_for_each(&port->dports, index, dport) 64 break; 65 single_port_map[0] = dport->port_id; 66 67 return add_hdm_decoder(port, &cxlsd->cxld, single_port_map); 68 } 69 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, CXL); 70 71 static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm) 72 { 73 u32 hdm_cap; 74 75 hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET); 76 cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap); 77 cxlhdm->target_count = 78 FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap); 79 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap)) 80 cxlhdm->interleave_mask |= GENMASK(11, 8); 81 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap)) 82 cxlhdm->interleave_mask |= GENMASK(14, 12); 83 } 84 85 static int map_hdm_decoder_regs(struct cxl_port *port, void __iomem *crb, 86 struct cxl_component_regs *regs) 87 { 88 struct cxl_register_map map = { 89 .resource = port->component_reg_phys, 90 .base = crb, 91 .max_size = CXL_COMPONENT_REG_BLOCK_SIZE, 92 }; 93 94 cxl_probe_component_regs(&port->dev, crb, &map.component_map); 95 if (!map.component_map.hdm_decoder.valid) { 96 dev_err(&port->dev, "HDM decoder registers invalid\n"); 97 return -ENXIO; 98 } 99 100 return cxl_map_component_regs(&port->dev, regs, &map, 101 BIT(CXL_CM_CAP_CAP_ID_HDM)); 102 } 103 104 static struct cxl_hdm *devm_cxl_setup_emulated_hdm(struct cxl_port *port, 105 struct cxl_endpoint_dvsec_info *info) 106 { 107 struct device *dev = &port->dev; 108 struct cxl_hdm *cxlhdm; 109 110 if (!info->mem_enabled) 111 return ERR_PTR(-ENODEV); 112 113 cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL); 114 if (!cxlhdm) 115 return ERR_PTR(-ENOMEM); 116 117 cxlhdm->port = port; 118 cxlhdm->decoder_count = info->ranges; 119 cxlhdm->target_count = info->ranges; 120 dev_set_drvdata(&port->dev, cxlhdm); 121 122 return cxlhdm; 123 } 124 125 /** 126 * devm_cxl_setup_hdm - map HDM decoder component registers 127 * @port: cxl_port to map 128 * @info: cached DVSEC range register info 129 */ 130 struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port, 131 struct cxl_endpoint_dvsec_info *info) 132 { 133 struct device *dev = &port->dev; 134 struct cxl_hdm *cxlhdm; 135 void __iomem *crb; 136 int rc; 137 138 cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL); 139 if (!cxlhdm) 140 return ERR_PTR(-ENOMEM); 141 142 cxlhdm->port = port; 143 crb = ioremap(port->component_reg_phys, CXL_COMPONENT_REG_BLOCK_SIZE); 144 if (!crb) { 145 if (info->mem_enabled) 146 return devm_cxl_setup_emulated_hdm(port, info); 147 148 dev_err(dev, "No component registers mapped\n"); 149 return ERR_PTR(-ENXIO); 150 } 151 152 rc = map_hdm_decoder_regs(port, crb, &cxlhdm->regs); 153 iounmap(crb); 154 if (rc) 155 return ERR_PTR(rc); 156 157 parse_hdm_decoder_caps(cxlhdm); 158 if (cxlhdm->decoder_count == 0) { 159 dev_err(dev, "Spec violation. Caps invalid\n"); 160 return ERR_PTR(-ENXIO); 161 } 162 163 dev_set_drvdata(dev, cxlhdm); 164 165 return cxlhdm; 166 } 167 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, CXL); 168 169 static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth) 170 { 171 unsigned long long start = r->start, end = r->end; 172 173 seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end, 174 r->name); 175 } 176 177 void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds) 178 { 179 struct resource *p1, *p2; 180 181 down_read(&cxl_dpa_rwsem); 182 for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) { 183 __cxl_dpa_debug(file, p1, 0); 184 for (p2 = p1->child; p2; p2 = p2->sibling) 185 __cxl_dpa_debug(file, p2, 1); 186 } 187 up_read(&cxl_dpa_rwsem); 188 } 189 EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, CXL); 190 191 /* 192 * Must be called in a context that synchronizes against this decoder's 193 * port ->remove() callback (like an endpoint decoder sysfs attribute) 194 */ 195 static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled) 196 { 197 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 198 struct cxl_port *port = cxled_to_port(cxled); 199 struct cxl_dev_state *cxlds = cxlmd->cxlds; 200 struct resource *res = cxled->dpa_res; 201 resource_size_t skip_start; 202 203 lockdep_assert_held_write(&cxl_dpa_rwsem); 204 205 /* save @skip_start, before @res is released */ 206 skip_start = res->start - cxled->skip; 207 __release_region(&cxlds->dpa_res, res->start, resource_size(res)); 208 if (cxled->skip) 209 __release_region(&cxlds->dpa_res, skip_start, cxled->skip); 210 cxled->skip = 0; 211 cxled->dpa_res = NULL; 212 put_device(&cxled->cxld.dev); 213 port->hdm_end--; 214 } 215 216 static void cxl_dpa_release(void *cxled) 217 { 218 down_write(&cxl_dpa_rwsem); 219 __cxl_dpa_release(cxled); 220 up_write(&cxl_dpa_rwsem); 221 } 222 223 /* 224 * Must be called from context that will not race port device 225 * unregistration, like decoder sysfs attribute methods 226 */ 227 static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled) 228 { 229 struct cxl_port *port = cxled_to_port(cxled); 230 231 lockdep_assert_held_write(&cxl_dpa_rwsem); 232 devm_remove_action(&port->dev, cxl_dpa_release, cxled); 233 __cxl_dpa_release(cxled); 234 } 235 236 static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, 237 resource_size_t base, resource_size_t len, 238 resource_size_t skipped) 239 { 240 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 241 struct cxl_port *port = cxled_to_port(cxled); 242 struct cxl_dev_state *cxlds = cxlmd->cxlds; 243 struct device *dev = &port->dev; 244 struct resource *res; 245 246 lockdep_assert_held_write(&cxl_dpa_rwsem); 247 248 if (!len) 249 goto success; 250 251 if (cxled->dpa_res) { 252 dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n", 253 port->id, cxled->cxld.id, cxled->dpa_res); 254 return -EBUSY; 255 } 256 257 if (port->hdm_end + 1 != cxled->cxld.id) { 258 /* 259 * Assumes alloc and commit order is always in hardware instance 260 * order per expectations from 8.2.5.12.20 Committing Decoder 261 * Programming that enforce decoder[m] committed before 262 * decoder[m+1] commit start. 263 */ 264 dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id, 265 cxled->cxld.id, port->id, port->hdm_end + 1); 266 return -EBUSY; 267 } 268 269 if (skipped) { 270 res = __request_region(&cxlds->dpa_res, base - skipped, skipped, 271 dev_name(&cxled->cxld.dev), 0); 272 if (!res) { 273 dev_dbg(dev, 274 "decoder%d.%d: failed to reserve skipped space\n", 275 port->id, cxled->cxld.id); 276 return -EBUSY; 277 } 278 } 279 res = __request_region(&cxlds->dpa_res, base, len, 280 dev_name(&cxled->cxld.dev), 0); 281 if (!res) { 282 dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n", 283 port->id, cxled->cxld.id); 284 if (skipped) 285 __release_region(&cxlds->dpa_res, base - skipped, 286 skipped); 287 return -EBUSY; 288 } 289 cxled->dpa_res = res; 290 cxled->skip = skipped; 291 292 if (resource_contains(&cxlds->pmem_res, res)) 293 cxled->mode = CXL_DECODER_PMEM; 294 else if (resource_contains(&cxlds->ram_res, res)) 295 cxled->mode = CXL_DECODER_RAM; 296 else { 297 dev_dbg(dev, "decoder%d.%d: %pr mixed\n", port->id, 298 cxled->cxld.id, cxled->dpa_res); 299 cxled->mode = CXL_DECODER_MIXED; 300 } 301 302 success: 303 port->hdm_end++; 304 get_device(&cxled->cxld.dev); 305 return 0; 306 } 307 308 static int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, 309 resource_size_t base, resource_size_t len, 310 resource_size_t skipped) 311 { 312 struct cxl_port *port = cxled_to_port(cxled); 313 int rc; 314 315 down_write(&cxl_dpa_rwsem); 316 rc = __cxl_dpa_reserve(cxled, base, len, skipped); 317 up_write(&cxl_dpa_rwsem); 318 319 if (rc) 320 return rc; 321 322 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled); 323 } 324 325 resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled) 326 { 327 resource_size_t size = 0; 328 329 down_read(&cxl_dpa_rwsem); 330 if (cxled->dpa_res) 331 size = resource_size(cxled->dpa_res); 332 up_read(&cxl_dpa_rwsem); 333 334 return size; 335 } 336 337 resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled) 338 { 339 resource_size_t base = -1; 340 341 down_read(&cxl_dpa_rwsem); 342 if (cxled->dpa_res) 343 base = cxled->dpa_res->start; 344 up_read(&cxl_dpa_rwsem); 345 346 return base; 347 } 348 349 int cxl_dpa_free(struct cxl_endpoint_decoder *cxled) 350 { 351 struct cxl_port *port = cxled_to_port(cxled); 352 struct device *dev = &cxled->cxld.dev; 353 int rc; 354 355 down_write(&cxl_dpa_rwsem); 356 if (!cxled->dpa_res) { 357 rc = 0; 358 goto out; 359 } 360 if (cxled->cxld.region) { 361 dev_dbg(dev, "decoder assigned to: %s\n", 362 dev_name(&cxled->cxld.region->dev)); 363 rc = -EBUSY; 364 goto out; 365 } 366 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) { 367 dev_dbg(dev, "decoder enabled\n"); 368 rc = -EBUSY; 369 goto out; 370 } 371 if (cxled->cxld.id != port->hdm_end) { 372 dev_dbg(dev, "expected decoder%d.%d\n", port->id, 373 port->hdm_end); 374 rc = -EBUSY; 375 goto out; 376 } 377 devm_cxl_dpa_release(cxled); 378 rc = 0; 379 out: 380 up_write(&cxl_dpa_rwsem); 381 return rc; 382 } 383 384 int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled, 385 enum cxl_decoder_mode mode) 386 { 387 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 388 struct cxl_dev_state *cxlds = cxlmd->cxlds; 389 struct device *dev = &cxled->cxld.dev; 390 int rc; 391 392 switch (mode) { 393 case CXL_DECODER_RAM: 394 case CXL_DECODER_PMEM: 395 break; 396 default: 397 dev_dbg(dev, "unsupported mode: %d\n", mode); 398 return -EINVAL; 399 } 400 401 down_write(&cxl_dpa_rwsem); 402 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) { 403 rc = -EBUSY; 404 goto out; 405 } 406 407 /* 408 * Only allow modes that are supported by the current partition 409 * configuration 410 */ 411 if (mode == CXL_DECODER_PMEM && !resource_size(&cxlds->pmem_res)) { 412 dev_dbg(dev, "no available pmem capacity\n"); 413 rc = -ENXIO; 414 goto out; 415 } 416 if (mode == CXL_DECODER_RAM && !resource_size(&cxlds->ram_res)) { 417 dev_dbg(dev, "no available ram capacity\n"); 418 rc = -ENXIO; 419 goto out; 420 } 421 422 cxled->mode = mode; 423 rc = 0; 424 out: 425 up_write(&cxl_dpa_rwsem); 426 427 return rc; 428 } 429 430 int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size) 431 { 432 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 433 resource_size_t free_ram_start, free_pmem_start; 434 struct cxl_port *port = cxled_to_port(cxled); 435 struct cxl_dev_state *cxlds = cxlmd->cxlds; 436 struct device *dev = &cxled->cxld.dev; 437 resource_size_t start, avail, skip; 438 struct resource *p, *last; 439 int rc; 440 441 down_write(&cxl_dpa_rwsem); 442 if (cxled->cxld.region) { 443 dev_dbg(dev, "decoder attached to %s\n", 444 dev_name(&cxled->cxld.region->dev)); 445 rc = -EBUSY; 446 goto out; 447 } 448 449 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) { 450 dev_dbg(dev, "decoder enabled\n"); 451 rc = -EBUSY; 452 goto out; 453 } 454 455 for (p = cxlds->ram_res.child, last = NULL; p; p = p->sibling) 456 last = p; 457 if (last) 458 free_ram_start = last->end + 1; 459 else 460 free_ram_start = cxlds->ram_res.start; 461 462 for (p = cxlds->pmem_res.child, last = NULL; p; p = p->sibling) 463 last = p; 464 if (last) 465 free_pmem_start = last->end + 1; 466 else 467 free_pmem_start = cxlds->pmem_res.start; 468 469 if (cxled->mode == CXL_DECODER_RAM) { 470 start = free_ram_start; 471 avail = cxlds->ram_res.end - start + 1; 472 skip = 0; 473 } else if (cxled->mode == CXL_DECODER_PMEM) { 474 resource_size_t skip_start, skip_end; 475 476 start = free_pmem_start; 477 avail = cxlds->pmem_res.end - start + 1; 478 skip_start = free_ram_start; 479 480 /* 481 * If some pmem is already allocated, then that allocation 482 * already handled the skip. 483 */ 484 if (cxlds->pmem_res.child && 485 skip_start == cxlds->pmem_res.child->start) 486 skip_end = skip_start - 1; 487 else 488 skip_end = start - 1; 489 skip = skip_end - skip_start + 1; 490 } else { 491 dev_dbg(dev, "mode not set\n"); 492 rc = -EINVAL; 493 goto out; 494 } 495 496 if (size > avail) { 497 dev_dbg(dev, "%pa exceeds available %s capacity: %pa\n", &size, 498 cxled->mode == CXL_DECODER_RAM ? "ram" : "pmem", 499 &avail); 500 rc = -ENOSPC; 501 goto out; 502 } 503 504 rc = __cxl_dpa_reserve(cxled, start, size, skip); 505 out: 506 up_write(&cxl_dpa_rwsem); 507 508 if (rc) 509 return rc; 510 511 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled); 512 } 513 514 static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl) 515 { 516 u16 eig; 517 u8 eiw; 518 519 /* 520 * Input validation ensures these warns never fire, but otherwise 521 * suppress unititalized variable usage warnings. 522 */ 523 if (WARN_ONCE(ways_to_eiw(cxld->interleave_ways, &eiw), 524 "invalid interleave_ways: %d\n", cxld->interleave_ways)) 525 return; 526 if (WARN_ONCE(granularity_to_eig(cxld->interleave_granularity, &eig), 527 "invalid interleave_granularity: %d\n", 528 cxld->interleave_granularity)) 529 return; 530 531 u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK); 532 u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK); 533 *ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT; 534 } 535 536 static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl) 537 { 538 u32p_replace_bits(ctrl, !!(cxld->target_type == 3), 539 CXL_HDM_DECODER0_CTRL_TYPE); 540 } 541 542 static int cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt) 543 { 544 struct cxl_dport **t = &cxlsd->target[0]; 545 int ways = cxlsd->cxld.interleave_ways; 546 547 if (dev_WARN_ONCE(&cxlsd->cxld.dev, 548 ways > 8 || ways > cxlsd->nr_targets, 549 "ways: %d overflows targets: %d\n", ways, 550 cxlsd->nr_targets)) 551 return -ENXIO; 552 553 *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id); 554 if (ways > 1) 555 *tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id); 556 if (ways > 2) 557 *tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id); 558 if (ways > 3) 559 *tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id); 560 if (ways > 4) 561 *tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id); 562 if (ways > 5) 563 *tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id); 564 if (ways > 6) 565 *tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id); 566 if (ways > 7) 567 *tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id); 568 569 return 0; 570 } 571 572 /* 573 * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set 574 * committed or error within 10ms, but just be generous with 20ms to account for 575 * clock skew and other marginal behavior 576 */ 577 #define COMMIT_TIMEOUT_MS 20 578 static int cxld_await_commit(void __iomem *hdm, int id) 579 { 580 u32 ctrl; 581 int i; 582 583 for (i = 0; i < COMMIT_TIMEOUT_MS; i++) { 584 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 585 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) { 586 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT; 587 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 588 return -EIO; 589 } 590 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl)) 591 return 0; 592 fsleep(1000); 593 } 594 595 return -ETIMEDOUT; 596 } 597 598 static int cxl_decoder_commit(struct cxl_decoder *cxld) 599 { 600 struct cxl_port *port = to_cxl_port(cxld->dev.parent); 601 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); 602 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 603 int id = cxld->id, rc; 604 u64 base, size; 605 u32 ctrl; 606 607 if (cxld->flags & CXL_DECODER_F_ENABLE) 608 return 0; 609 610 if (port->commit_end + 1 != id) { 611 dev_dbg(&port->dev, 612 "%s: out of order commit, expected decoder%d.%d\n", 613 dev_name(&cxld->dev), port->id, port->commit_end + 1); 614 return -EBUSY; 615 } 616 617 down_read(&cxl_dpa_rwsem); 618 /* common decoder settings */ 619 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id)); 620 cxld_set_interleave(cxld, &ctrl); 621 cxld_set_type(cxld, &ctrl); 622 base = cxld->hpa_range.start; 623 size = range_len(&cxld->hpa_range); 624 625 writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id)); 626 writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id)); 627 writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id)); 628 writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id)); 629 630 if (is_switch_decoder(&cxld->dev)) { 631 struct cxl_switch_decoder *cxlsd = 632 to_cxl_switch_decoder(&cxld->dev); 633 void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id); 634 void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id); 635 u64 targets; 636 637 rc = cxlsd_set_targets(cxlsd, &targets); 638 if (rc) { 639 dev_dbg(&port->dev, "%s: target configuration error\n", 640 dev_name(&cxld->dev)); 641 goto err; 642 } 643 644 writel(upper_32_bits(targets), tl_hi); 645 writel(lower_32_bits(targets), tl_lo); 646 } else { 647 struct cxl_endpoint_decoder *cxled = 648 to_cxl_endpoint_decoder(&cxld->dev); 649 void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id); 650 void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id); 651 652 writel(upper_32_bits(cxled->skip), sk_hi); 653 writel(lower_32_bits(cxled->skip), sk_lo); 654 } 655 656 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 657 up_read(&cxl_dpa_rwsem); 658 659 port->commit_end++; 660 rc = cxld_await_commit(hdm, cxld->id); 661 err: 662 if (rc) { 663 dev_dbg(&port->dev, "%s: error %d committing decoder\n", 664 dev_name(&cxld->dev), rc); 665 cxld->reset(cxld); 666 return rc; 667 } 668 cxld->flags |= CXL_DECODER_F_ENABLE; 669 670 return 0; 671 } 672 673 static int cxl_decoder_reset(struct cxl_decoder *cxld) 674 { 675 struct cxl_port *port = to_cxl_port(cxld->dev.parent); 676 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); 677 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 678 int id = cxld->id; 679 u32 ctrl; 680 681 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0) 682 return 0; 683 684 if (port->commit_end != id) { 685 dev_dbg(&port->dev, 686 "%s: out of order reset, expected decoder%d.%d\n", 687 dev_name(&cxld->dev), port->id, port->commit_end); 688 return -EBUSY; 689 } 690 691 down_read(&cxl_dpa_rwsem); 692 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 693 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT; 694 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 695 696 writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id)); 697 writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id)); 698 writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id)); 699 writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id)); 700 up_read(&cxl_dpa_rwsem); 701 702 port->commit_end--; 703 cxld->flags &= ~CXL_DECODER_F_ENABLE; 704 705 return 0; 706 } 707 708 static int cxl_setup_hdm_decoder_from_dvsec(struct cxl_port *port, 709 struct cxl_decoder *cxld, int which, 710 struct cxl_endpoint_dvsec_info *info) 711 { 712 if (!is_cxl_endpoint(port)) 713 return -EOPNOTSUPP; 714 715 if (!range_len(&info->dvsec_range[which])) 716 return -ENOENT; 717 718 cxld->target_type = CXL_DECODER_EXPANDER; 719 cxld->commit = NULL; 720 cxld->reset = NULL; 721 cxld->hpa_range = info->dvsec_range[which]; 722 723 /* 724 * Set the emulated decoder as locked pending additional support to 725 * change the range registers at run time. 726 */ 727 cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK; 728 port->commit_end = cxld->id; 729 730 return 0; 731 } 732 733 static bool should_emulate_decoders(struct cxl_port *port) 734 { 735 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); 736 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 737 u32 ctrl; 738 int i; 739 740 if (!is_cxl_endpoint(cxlhdm->port)) 741 return false; 742 743 if (!hdm) 744 return true; 745 746 /* 747 * If any decoders are committed already, there should not be any 748 * emulated DVSEC decoders. 749 */ 750 for (i = 0; i < cxlhdm->decoder_count; i++) { 751 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i)); 752 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl)) 753 return false; 754 } 755 756 return true; 757 } 758 759 static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, 760 int *target_map, void __iomem *hdm, int which, 761 u64 *dpa_base, struct cxl_endpoint_dvsec_info *info) 762 { 763 struct cxl_endpoint_decoder *cxled = NULL; 764 u64 size, base, skip, dpa_size; 765 bool committed; 766 u32 remainder; 767 int i, rc; 768 u32 ctrl; 769 union { 770 u64 value; 771 unsigned char target_id[8]; 772 } target_list; 773 774 if (should_emulate_decoders(port)) 775 return cxl_setup_hdm_decoder_from_dvsec(port, cxld, which, info); 776 777 if (is_endpoint_decoder(&cxld->dev)) 778 cxled = to_cxl_endpoint_decoder(&cxld->dev); 779 780 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which)); 781 base = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which)); 782 size = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which)); 783 committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED); 784 cxld->commit = cxl_decoder_commit; 785 cxld->reset = cxl_decoder_reset; 786 787 if (!committed) 788 size = 0; 789 if (base == U64_MAX || size == U64_MAX) { 790 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n", 791 port->id, cxld->id); 792 return -ENXIO; 793 } 794 795 cxld->hpa_range = (struct range) { 796 .start = base, 797 .end = base + size - 1, 798 }; 799 800 if (cxled && !committed && range_len(&info->dvsec_range[which])) 801 return cxl_setup_hdm_decoder_from_dvsec(port, cxld, which, info); 802 803 /* decoders are enabled if committed */ 804 if (committed) { 805 cxld->flags |= CXL_DECODER_F_ENABLE; 806 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK) 807 cxld->flags |= CXL_DECODER_F_LOCK; 808 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl)) 809 cxld->target_type = CXL_DECODER_EXPANDER; 810 else 811 cxld->target_type = CXL_DECODER_ACCELERATOR; 812 if (cxld->id != port->commit_end + 1) { 813 dev_warn(&port->dev, 814 "decoder%d.%d: Committed out of order\n", 815 port->id, cxld->id); 816 return -ENXIO; 817 } 818 port->commit_end = cxld->id; 819 } else { 820 /* unless / until type-2 drivers arrive, assume type-3 */ 821 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl) == 0) { 822 ctrl |= CXL_HDM_DECODER0_CTRL_TYPE; 823 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which)); 824 } 825 cxld->target_type = CXL_DECODER_EXPANDER; 826 } 827 rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl), 828 &cxld->interleave_ways); 829 if (rc) { 830 dev_warn(&port->dev, 831 "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n", 832 port->id, cxld->id, ctrl); 833 return rc; 834 } 835 rc = eig_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl), 836 &cxld->interleave_granularity); 837 if (rc) 838 return rc; 839 840 if (!cxled) { 841 target_list.value = 842 ioread64_hi_lo(hdm + CXL_HDM_DECODER0_TL_LOW(which)); 843 for (i = 0; i < cxld->interleave_ways; i++) 844 target_map[i] = target_list.target_id[i]; 845 846 return 0; 847 } 848 849 if (!committed) 850 return 0; 851 852 dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder); 853 if (remainder) { 854 dev_err(&port->dev, 855 "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n", 856 port->id, cxld->id, size, cxld->interleave_ways); 857 return -ENXIO; 858 } 859 skip = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_SKIP_LOW(which)); 860 rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip); 861 if (rc) { 862 dev_err(&port->dev, 863 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)", 864 port->id, cxld->id, *dpa_base, 865 *dpa_base + dpa_size + skip - 1, rc); 866 return rc; 867 } 868 *dpa_base += dpa_size + skip; 869 return 0; 870 } 871 872 static void cxl_settle_decoders(struct cxl_hdm *cxlhdm) 873 { 874 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 875 int committed, i; 876 u32 ctrl; 877 878 if (!hdm) 879 return; 880 881 /* 882 * Since the register resource was recently claimed via request_region() 883 * be careful about trusting the "not-committed" status until the commit 884 * timeout has elapsed. The commit timeout is 10ms (CXL 2.0 885 * 8.2.5.12.20), but double it to be tolerant of any clock skew between 886 * host and target. 887 */ 888 for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) { 889 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i)); 890 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED) 891 committed++; 892 } 893 894 /* ensure that future checks of committed can be trusted */ 895 if (committed != cxlhdm->decoder_count) 896 msleep(20); 897 } 898 899 /** 900 * devm_cxl_enumerate_decoders - add decoder objects per HDM register set 901 * @cxlhdm: Structure to populate with HDM capabilities 902 * @info: cached DVSEC range register info 903 */ 904 int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm, 905 struct cxl_endpoint_dvsec_info *info) 906 { 907 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 908 struct cxl_port *port = cxlhdm->port; 909 int i; 910 u64 dpa_base = 0; 911 912 cxl_settle_decoders(cxlhdm); 913 914 for (i = 0; i < cxlhdm->decoder_count; i++) { 915 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 }; 916 int rc, target_count = cxlhdm->target_count; 917 struct cxl_decoder *cxld; 918 919 if (is_cxl_endpoint(port)) { 920 struct cxl_endpoint_decoder *cxled; 921 922 cxled = cxl_endpoint_decoder_alloc(port); 923 if (IS_ERR(cxled)) { 924 dev_warn(&port->dev, 925 "Failed to allocate the decoder\n"); 926 return PTR_ERR(cxled); 927 } 928 cxld = &cxled->cxld; 929 } else { 930 struct cxl_switch_decoder *cxlsd; 931 932 cxlsd = cxl_switch_decoder_alloc(port, target_count); 933 if (IS_ERR(cxlsd)) { 934 dev_warn(&port->dev, 935 "Failed to allocate the decoder\n"); 936 return PTR_ERR(cxlsd); 937 } 938 cxld = &cxlsd->cxld; 939 } 940 941 rc = init_hdm_decoder(port, cxld, target_map, hdm, i, 942 &dpa_base, info); 943 if (rc) { 944 put_device(&cxld->dev); 945 return rc; 946 } 947 rc = add_hdm_decoder(port, cxld, target_map); 948 if (rc) { 949 dev_warn(&port->dev, 950 "Failed to add decoder to port\n"); 951 return rc; 952 } 953 } 954 955 return 0; 956 } 957 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, CXL); 958