1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */ 3 #include <linux/seq_file.h> 4 #include <linux/device.h> 5 #include <linux/delay.h> 6 7 #include "cxlmem.h" 8 #include "core.h" 9 10 /** 11 * DOC: cxl core hdm 12 * 13 * Compute Express Link Host Managed Device Memory, starting with the 14 * CXL 2.0 specification, is managed by an array of HDM Decoder register 15 * instances per CXL port and per CXL endpoint. Define common helpers 16 * for enumerating these registers and capabilities. 17 */ 18 19 DECLARE_RWSEM(cxl_dpa_rwsem); 20 21 static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, 22 int *target_map) 23 { 24 int rc; 25 26 rc = cxl_decoder_add_locked(cxld, target_map); 27 if (rc) { 28 put_device(&cxld->dev); 29 dev_err(&port->dev, "Failed to add decoder\n"); 30 return rc; 31 } 32 33 rc = cxl_decoder_autoremove(&port->dev, cxld); 34 if (rc) 35 return rc; 36 37 dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev)); 38 39 return 0; 40 } 41 42 /* 43 * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure) 44 * single ported host-bridges need not publish a decoder capability when a 45 * passthrough decode can be assumed, i.e. all transactions that the uport sees 46 * are claimed and passed to the single dport. Disable the range until the first 47 * CXL region is enumerated / activated. 48 */ 49 int devm_cxl_add_passthrough_decoder(struct cxl_port *port) 50 { 51 struct cxl_switch_decoder *cxlsd; 52 struct cxl_dport *dport = NULL; 53 int single_port_map[1]; 54 unsigned long index; 55 56 cxlsd = cxl_switch_decoder_alloc(port, 1); 57 if (IS_ERR(cxlsd)) 58 return PTR_ERR(cxlsd); 59 60 device_lock_assert(&port->dev); 61 62 xa_for_each(&port->dports, index, dport) 63 break; 64 single_port_map[0] = dport->port_id; 65 66 return add_hdm_decoder(port, &cxlsd->cxld, single_port_map); 67 } 68 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, CXL); 69 70 static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm) 71 { 72 u32 hdm_cap; 73 74 hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET); 75 cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap); 76 cxlhdm->target_count = 77 FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap); 78 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap)) 79 cxlhdm->interleave_mask |= GENMASK(11, 8); 80 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap)) 81 cxlhdm->interleave_mask |= GENMASK(14, 12); 82 } 83 84 static int map_hdm_decoder_regs(struct cxl_port *port, void __iomem *crb, 85 struct cxl_component_regs *regs) 86 { 87 struct cxl_register_map map = { 88 .host = &port->dev, 89 .resource = port->component_reg_phys, 90 .base = crb, 91 .max_size = CXL_COMPONENT_REG_BLOCK_SIZE, 92 }; 93 94 cxl_probe_component_regs(&port->dev, crb, &map.component_map); 95 if (!map.component_map.hdm_decoder.valid) { 96 dev_dbg(&port->dev, "HDM decoder registers not implemented\n"); 97 /* unique error code to indicate no HDM decoder capability */ 98 return -ENODEV; 99 } 100 101 return cxl_map_component_regs(&map, regs, BIT(CXL_CM_CAP_CAP_ID_HDM)); 102 } 103 104 static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info) 105 { 106 struct cxl_hdm *cxlhdm; 107 void __iomem *hdm; 108 u32 ctrl; 109 int i; 110 111 if (!info) 112 return false; 113 114 cxlhdm = dev_get_drvdata(&info->port->dev); 115 hdm = cxlhdm->regs.hdm_decoder; 116 117 if (!hdm) 118 return true; 119 120 /* 121 * If HDM decoders are present and the driver is in control of 122 * Mem_Enable skip DVSEC based emulation 123 */ 124 if (!info->mem_enabled) 125 return false; 126 127 /* 128 * If any decoders are committed already, there should not be any 129 * emulated DVSEC decoders. 130 */ 131 for (i = 0; i < cxlhdm->decoder_count; i++) { 132 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i)); 133 dev_dbg(&info->port->dev, 134 "decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n", 135 info->port->id, i, 136 FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl), 137 readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)), 138 readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)), 139 readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)), 140 readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i))); 141 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl)) 142 return false; 143 } 144 145 return true; 146 } 147 148 /** 149 * devm_cxl_setup_hdm - map HDM decoder component registers 150 * @port: cxl_port to map 151 * @info: cached DVSEC range register info 152 */ 153 struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port, 154 struct cxl_endpoint_dvsec_info *info) 155 { 156 struct device *dev = &port->dev; 157 struct cxl_hdm *cxlhdm; 158 void __iomem *crb; 159 int rc; 160 161 cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL); 162 if (!cxlhdm) 163 return ERR_PTR(-ENOMEM); 164 cxlhdm->port = port; 165 dev_set_drvdata(dev, cxlhdm); 166 167 crb = ioremap(port->component_reg_phys, CXL_COMPONENT_REG_BLOCK_SIZE); 168 if (!crb && info && info->mem_enabled) { 169 cxlhdm->decoder_count = info->ranges; 170 return cxlhdm; 171 } else if (!crb) { 172 dev_err(dev, "No component registers mapped\n"); 173 return ERR_PTR(-ENXIO); 174 } 175 176 rc = map_hdm_decoder_regs(port, crb, &cxlhdm->regs); 177 iounmap(crb); 178 if (rc) 179 return ERR_PTR(rc); 180 181 parse_hdm_decoder_caps(cxlhdm); 182 if (cxlhdm->decoder_count == 0) { 183 dev_err(dev, "Spec violation. Caps invalid\n"); 184 return ERR_PTR(-ENXIO); 185 } 186 187 /* 188 * Now that the hdm capability is parsed, decide if range 189 * register emulation is needed and fixup cxlhdm accordingly. 190 */ 191 if (should_emulate_decoders(info)) { 192 dev_dbg(dev, "Fallback map %d range register%s\n", info->ranges, 193 info->ranges > 1 ? "s" : ""); 194 cxlhdm->decoder_count = info->ranges; 195 } 196 197 return cxlhdm; 198 } 199 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, CXL); 200 201 static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth) 202 { 203 unsigned long long start = r->start, end = r->end; 204 205 seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end, 206 r->name); 207 } 208 209 void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds) 210 { 211 struct resource *p1, *p2; 212 213 down_read(&cxl_dpa_rwsem); 214 for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) { 215 __cxl_dpa_debug(file, p1, 0); 216 for (p2 = p1->child; p2; p2 = p2->sibling) 217 __cxl_dpa_debug(file, p2, 1); 218 } 219 up_read(&cxl_dpa_rwsem); 220 } 221 EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, CXL); 222 223 /* 224 * Must be called in a context that synchronizes against this decoder's 225 * port ->remove() callback (like an endpoint decoder sysfs attribute) 226 */ 227 static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled) 228 { 229 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 230 struct cxl_port *port = cxled_to_port(cxled); 231 struct cxl_dev_state *cxlds = cxlmd->cxlds; 232 struct resource *res = cxled->dpa_res; 233 resource_size_t skip_start; 234 235 lockdep_assert_held_write(&cxl_dpa_rwsem); 236 237 /* save @skip_start, before @res is released */ 238 skip_start = res->start - cxled->skip; 239 __release_region(&cxlds->dpa_res, res->start, resource_size(res)); 240 if (cxled->skip) 241 __release_region(&cxlds->dpa_res, skip_start, cxled->skip); 242 cxled->skip = 0; 243 cxled->dpa_res = NULL; 244 put_device(&cxled->cxld.dev); 245 port->hdm_end--; 246 } 247 248 static void cxl_dpa_release(void *cxled) 249 { 250 down_write(&cxl_dpa_rwsem); 251 __cxl_dpa_release(cxled); 252 up_write(&cxl_dpa_rwsem); 253 } 254 255 /* 256 * Must be called from context that will not race port device 257 * unregistration, like decoder sysfs attribute methods 258 */ 259 static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled) 260 { 261 struct cxl_port *port = cxled_to_port(cxled); 262 263 lockdep_assert_held_write(&cxl_dpa_rwsem); 264 devm_remove_action(&port->dev, cxl_dpa_release, cxled); 265 __cxl_dpa_release(cxled); 266 } 267 268 static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, 269 resource_size_t base, resource_size_t len, 270 resource_size_t skipped) 271 { 272 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 273 struct cxl_port *port = cxled_to_port(cxled); 274 struct cxl_dev_state *cxlds = cxlmd->cxlds; 275 struct device *dev = &port->dev; 276 struct resource *res; 277 278 lockdep_assert_held_write(&cxl_dpa_rwsem); 279 280 if (!len) { 281 dev_warn(dev, "decoder%d.%d: empty reservation attempted\n", 282 port->id, cxled->cxld.id); 283 return -EINVAL; 284 } 285 286 if (cxled->dpa_res) { 287 dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n", 288 port->id, cxled->cxld.id, cxled->dpa_res); 289 return -EBUSY; 290 } 291 292 if (port->hdm_end + 1 != cxled->cxld.id) { 293 /* 294 * Assumes alloc and commit order is always in hardware instance 295 * order per expectations from 8.2.5.12.20 Committing Decoder 296 * Programming that enforce decoder[m] committed before 297 * decoder[m+1] commit start. 298 */ 299 dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id, 300 cxled->cxld.id, port->id, port->hdm_end + 1); 301 return -EBUSY; 302 } 303 304 if (skipped) { 305 res = __request_region(&cxlds->dpa_res, base - skipped, skipped, 306 dev_name(&cxled->cxld.dev), 0); 307 if (!res) { 308 dev_dbg(dev, 309 "decoder%d.%d: failed to reserve skipped space\n", 310 port->id, cxled->cxld.id); 311 return -EBUSY; 312 } 313 } 314 res = __request_region(&cxlds->dpa_res, base, len, 315 dev_name(&cxled->cxld.dev), 0); 316 if (!res) { 317 dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n", 318 port->id, cxled->cxld.id); 319 if (skipped) 320 __release_region(&cxlds->dpa_res, base - skipped, 321 skipped); 322 return -EBUSY; 323 } 324 cxled->dpa_res = res; 325 cxled->skip = skipped; 326 327 if (resource_contains(&cxlds->pmem_res, res)) 328 cxled->mode = CXL_DECODER_PMEM; 329 else if (resource_contains(&cxlds->ram_res, res)) 330 cxled->mode = CXL_DECODER_RAM; 331 else { 332 dev_dbg(dev, "decoder%d.%d: %pr mixed\n", port->id, 333 cxled->cxld.id, cxled->dpa_res); 334 cxled->mode = CXL_DECODER_MIXED; 335 } 336 337 port->hdm_end++; 338 get_device(&cxled->cxld.dev); 339 return 0; 340 } 341 342 int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, 343 resource_size_t base, resource_size_t len, 344 resource_size_t skipped) 345 { 346 struct cxl_port *port = cxled_to_port(cxled); 347 int rc; 348 349 down_write(&cxl_dpa_rwsem); 350 rc = __cxl_dpa_reserve(cxled, base, len, skipped); 351 up_write(&cxl_dpa_rwsem); 352 353 if (rc) 354 return rc; 355 356 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled); 357 } 358 EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, CXL); 359 360 resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled) 361 { 362 resource_size_t size = 0; 363 364 down_read(&cxl_dpa_rwsem); 365 if (cxled->dpa_res) 366 size = resource_size(cxled->dpa_res); 367 up_read(&cxl_dpa_rwsem); 368 369 return size; 370 } 371 372 resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled) 373 { 374 resource_size_t base = -1; 375 376 down_read(&cxl_dpa_rwsem); 377 if (cxled->dpa_res) 378 base = cxled->dpa_res->start; 379 up_read(&cxl_dpa_rwsem); 380 381 return base; 382 } 383 384 int cxl_dpa_free(struct cxl_endpoint_decoder *cxled) 385 { 386 struct cxl_port *port = cxled_to_port(cxled); 387 struct device *dev = &cxled->cxld.dev; 388 int rc; 389 390 down_write(&cxl_dpa_rwsem); 391 if (!cxled->dpa_res) { 392 rc = 0; 393 goto out; 394 } 395 if (cxled->cxld.region) { 396 dev_dbg(dev, "decoder assigned to: %s\n", 397 dev_name(&cxled->cxld.region->dev)); 398 rc = -EBUSY; 399 goto out; 400 } 401 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) { 402 dev_dbg(dev, "decoder enabled\n"); 403 rc = -EBUSY; 404 goto out; 405 } 406 if (cxled->cxld.id != port->hdm_end) { 407 dev_dbg(dev, "expected decoder%d.%d\n", port->id, 408 port->hdm_end); 409 rc = -EBUSY; 410 goto out; 411 } 412 devm_cxl_dpa_release(cxled); 413 rc = 0; 414 out: 415 up_write(&cxl_dpa_rwsem); 416 return rc; 417 } 418 419 int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled, 420 enum cxl_decoder_mode mode) 421 { 422 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 423 struct cxl_dev_state *cxlds = cxlmd->cxlds; 424 struct device *dev = &cxled->cxld.dev; 425 int rc; 426 427 switch (mode) { 428 case CXL_DECODER_RAM: 429 case CXL_DECODER_PMEM: 430 break; 431 default: 432 dev_dbg(dev, "unsupported mode: %d\n", mode); 433 return -EINVAL; 434 } 435 436 down_write(&cxl_dpa_rwsem); 437 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) { 438 rc = -EBUSY; 439 goto out; 440 } 441 442 /* 443 * Only allow modes that are supported by the current partition 444 * configuration 445 */ 446 if (mode == CXL_DECODER_PMEM && !resource_size(&cxlds->pmem_res)) { 447 dev_dbg(dev, "no available pmem capacity\n"); 448 rc = -ENXIO; 449 goto out; 450 } 451 if (mode == CXL_DECODER_RAM && !resource_size(&cxlds->ram_res)) { 452 dev_dbg(dev, "no available ram capacity\n"); 453 rc = -ENXIO; 454 goto out; 455 } 456 457 cxled->mode = mode; 458 rc = 0; 459 out: 460 up_write(&cxl_dpa_rwsem); 461 462 return rc; 463 } 464 465 int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size) 466 { 467 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 468 resource_size_t free_ram_start, free_pmem_start; 469 struct cxl_port *port = cxled_to_port(cxled); 470 struct cxl_dev_state *cxlds = cxlmd->cxlds; 471 struct device *dev = &cxled->cxld.dev; 472 resource_size_t start, avail, skip; 473 struct resource *p, *last; 474 int rc; 475 476 down_write(&cxl_dpa_rwsem); 477 if (cxled->cxld.region) { 478 dev_dbg(dev, "decoder attached to %s\n", 479 dev_name(&cxled->cxld.region->dev)); 480 rc = -EBUSY; 481 goto out; 482 } 483 484 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) { 485 dev_dbg(dev, "decoder enabled\n"); 486 rc = -EBUSY; 487 goto out; 488 } 489 490 for (p = cxlds->ram_res.child, last = NULL; p; p = p->sibling) 491 last = p; 492 if (last) 493 free_ram_start = last->end + 1; 494 else 495 free_ram_start = cxlds->ram_res.start; 496 497 for (p = cxlds->pmem_res.child, last = NULL; p; p = p->sibling) 498 last = p; 499 if (last) 500 free_pmem_start = last->end + 1; 501 else 502 free_pmem_start = cxlds->pmem_res.start; 503 504 if (cxled->mode == CXL_DECODER_RAM) { 505 start = free_ram_start; 506 avail = cxlds->ram_res.end - start + 1; 507 skip = 0; 508 } else if (cxled->mode == CXL_DECODER_PMEM) { 509 resource_size_t skip_start, skip_end; 510 511 start = free_pmem_start; 512 avail = cxlds->pmem_res.end - start + 1; 513 skip_start = free_ram_start; 514 515 /* 516 * If some pmem is already allocated, then that allocation 517 * already handled the skip. 518 */ 519 if (cxlds->pmem_res.child && 520 skip_start == cxlds->pmem_res.child->start) 521 skip_end = skip_start - 1; 522 else 523 skip_end = start - 1; 524 skip = skip_end - skip_start + 1; 525 } else { 526 dev_dbg(dev, "mode not set\n"); 527 rc = -EINVAL; 528 goto out; 529 } 530 531 if (size > avail) { 532 dev_dbg(dev, "%pa exceeds available %s capacity: %pa\n", &size, 533 cxled->mode == CXL_DECODER_RAM ? "ram" : "pmem", 534 &avail); 535 rc = -ENOSPC; 536 goto out; 537 } 538 539 rc = __cxl_dpa_reserve(cxled, start, size, skip); 540 out: 541 up_write(&cxl_dpa_rwsem); 542 543 if (rc) 544 return rc; 545 546 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled); 547 } 548 549 static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl) 550 { 551 u16 eig; 552 u8 eiw; 553 554 /* 555 * Input validation ensures these warns never fire, but otherwise 556 * suppress unititalized variable usage warnings. 557 */ 558 if (WARN_ONCE(ways_to_eiw(cxld->interleave_ways, &eiw), 559 "invalid interleave_ways: %d\n", cxld->interleave_ways)) 560 return; 561 if (WARN_ONCE(granularity_to_eig(cxld->interleave_granularity, &eig), 562 "invalid interleave_granularity: %d\n", 563 cxld->interleave_granularity)) 564 return; 565 566 u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK); 567 u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK); 568 *ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT; 569 } 570 571 static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl) 572 { 573 u32p_replace_bits(ctrl, 574 !!(cxld->target_type == CXL_DECODER_HOSTONLYMEM), 575 CXL_HDM_DECODER0_CTRL_HOSTONLY); 576 } 577 578 static void cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt) 579 { 580 struct cxl_dport **t = &cxlsd->target[0]; 581 int ways = cxlsd->cxld.interleave_ways; 582 583 *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id); 584 if (ways > 1) 585 *tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id); 586 if (ways > 2) 587 *tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id); 588 if (ways > 3) 589 *tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id); 590 if (ways > 4) 591 *tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id); 592 if (ways > 5) 593 *tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id); 594 if (ways > 6) 595 *tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id); 596 if (ways > 7) 597 *tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id); 598 } 599 600 /* 601 * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set 602 * committed or error within 10ms, but just be generous with 20ms to account for 603 * clock skew and other marginal behavior 604 */ 605 #define COMMIT_TIMEOUT_MS 20 606 static int cxld_await_commit(void __iomem *hdm, int id) 607 { 608 u32 ctrl; 609 int i; 610 611 for (i = 0; i < COMMIT_TIMEOUT_MS; i++) { 612 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 613 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) { 614 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT; 615 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 616 return -EIO; 617 } 618 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl)) 619 return 0; 620 fsleep(1000); 621 } 622 623 return -ETIMEDOUT; 624 } 625 626 static int cxl_decoder_commit(struct cxl_decoder *cxld) 627 { 628 struct cxl_port *port = to_cxl_port(cxld->dev.parent); 629 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); 630 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 631 int id = cxld->id, rc; 632 u64 base, size; 633 u32 ctrl; 634 635 if (cxld->flags & CXL_DECODER_F_ENABLE) 636 return 0; 637 638 if (port->commit_end + 1 != id) { 639 dev_dbg(&port->dev, 640 "%s: out of order commit, expected decoder%d.%d\n", 641 dev_name(&cxld->dev), port->id, port->commit_end + 1); 642 return -EBUSY; 643 } 644 645 /* 646 * For endpoint decoders hosted on CXL memory devices that 647 * support the sanitize operation, make sure sanitize is not in-flight. 648 */ 649 if (is_endpoint_decoder(&cxld->dev)) { 650 struct cxl_endpoint_decoder *cxled = 651 to_cxl_endpoint_decoder(&cxld->dev); 652 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 653 struct cxl_memdev_state *mds = 654 to_cxl_memdev_state(cxlmd->cxlds); 655 656 if (mds && mds->security.sanitize_active) { 657 dev_dbg(&cxlmd->dev, 658 "attempted to commit %s during sanitize\n", 659 dev_name(&cxld->dev)); 660 return -EBUSY; 661 } 662 } 663 664 down_read(&cxl_dpa_rwsem); 665 /* common decoder settings */ 666 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id)); 667 cxld_set_interleave(cxld, &ctrl); 668 cxld_set_type(cxld, &ctrl); 669 base = cxld->hpa_range.start; 670 size = range_len(&cxld->hpa_range); 671 672 writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id)); 673 writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id)); 674 writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id)); 675 writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id)); 676 677 if (is_switch_decoder(&cxld->dev)) { 678 struct cxl_switch_decoder *cxlsd = 679 to_cxl_switch_decoder(&cxld->dev); 680 void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id); 681 void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id); 682 u64 targets; 683 684 cxlsd_set_targets(cxlsd, &targets); 685 writel(upper_32_bits(targets), tl_hi); 686 writel(lower_32_bits(targets), tl_lo); 687 } else { 688 struct cxl_endpoint_decoder *cxled = 689 to_cxl_endpoint_decoder(&cxld->dev); 690 void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id); 691 void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id); 692 693 writel(upper_32_bits(cxled->skip), sk_hi); 694 writel(lower_32_bits(cxled->skip), sk_lo); 695 } 696 697 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 698 up_read(&cxl_dpa_rwsem); 699 700 port->commit_end++; 701 rc = cxld_await_commit(hdm, cxld->id); 702 if (rc) { 703 dev_dbg(&port->dev, "%s: error %d committing decoder\n", 704 dev_name(&cxld->dev), rc); 705 cxld->reset(cxld); 706 return rc; 707 } 708 cxld->flags |= CXL_DECODER_F_ENABLE; 709 710 return 0; 711 } 712 713 static int cxl_decoder_reset(struct cxl_decoder *cxld) 714 { 715 struct cxl_port *port = to_cxl_port(cxld->dev.parent); 716 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); 717 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 718 int id = cxld->id; 719 u32 ctrl; 720 721 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0) 722 return 0; 723 724 if (port->commit_end != id) { 725 dev_dbg(&port->dev, 726 "%s: out of order reset, expected decoder%d.%d\n", 727 dev_name(&cxld->dev), port->id, port->commit_end); 728 return -EBUSY; 729 } 730 731 down_read(&cxl_dpa_rwsem); 732 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 733 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT; 734 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 735 736 writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id)); 737 writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id)); 738 writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id)); 739 writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id)); 740 up_read(&cxl_dpa_rwsem); 741 742 port->commit_end--; 743 cxld->flags &= ~CXL_DECODER_F_ENABLE; 744 745 /* Userspace is now responsible for reconfiguring this decoder */ 746 if (is_endpoint_decoder(&cxld->dev)) { 747 struct cxl_endpoint_decoder *cxled; 748 749 cxled = to_cxl_endpoint_decoder(&cxld->dev); 750 cxled->state = CXL_DECODER_STATE_MANUAL; 751 } 752 753 return 0; 754 } 755 756 static int cxl_setup_hdm_decoder_from_dvsec( 757 struct cxl_port *port, struct cxl_decoder *cxld, u64 *dpa_base, 758 int which, struct cxl_endpoint_dvsec_info *info) 759 { 760 struct cxl_endpoint_decoder *cxled; 761 u64 len; 762 int rc; 763 764 if (!is_cxl_endpoint(port)) 765 return -EOPNOTSUPP; 766 767 cxled = to_cxl_endpoint_decoder(&cxld->dev); 768 len = range_len(&info->dvsec_range[which]); 769 if (!len) 770 return -ENOENT; 771 772 cxld->target_type = CXL_DECODER_HOSTONLYMEM; 773 cxld->commit = NULL; 774 cxld->reset = NULL; 775 cxld->hpa_range = info->dvsec_range[which]; 776 777 /* 778 * Set the emulated decoder as locked pending additional support to 779 * change the range registers at run time. 780 */ 781 cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK; 782 port->commit_end = cxld->id; 783 784 rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0); 785 if (rc) { 786 dev_err(&port->dev, 787 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)", 788 port->id, cxld->id, *dpa_base, *dpa_base + len - 1, rc); 789 return rc; 790 } 791 *dpa_base += len; 792 cxled->state = CXL_DECODER_STATE_AUTO; 793 794 return 0; 795 } 796 797 static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, 798 int *target_map, void __iomem *hdm, int which, 799 u64 *dpa_base, struct cxl_endpoint_dvsec_info *info) 800 { 801 struct cxl_endpoint_decoder *cxled = NULL; 802 u64 size, base, skip, dpa_size, lo, hi; 803 bool committed; 804 u32 remainder; 805 int i, rc; 806 u32 ctrl; 807 union { 808 u64 value; 809 unsigned char target_id[8]; 810 } target_list; 811 812 if (should_emulate_decoders(info)) 813 return cxl_setup_hdm_decoder_from_dvsec(port, cxld, dpa_base, 814 which, info); 815 816 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which)); 817 lo = readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which)); 818 hi = readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(which)); 819 base = (hi << 32) + lo; 820 lo = readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which)); 821 hi = readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(which)); 822 size = (hi << 32) + lo; 823 committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED); 824 cxld->commit = cxl_decoder_commit; 825 cxld->reset = cxl_decoder_reset; 826 827 if (!committed) 828 size = 0; 829 if (base == U64_MAX || size == U64_MAX) { 830 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n", 831 port->id, cxld->id); 832 return -ENXIO; 833 } 834 835 if (info) 836 cxled = to_cxl_endpoint_decoder(&cxld->dev); 837 cxld->hpa_range = (struct range) { 838 .start = base, 839 .end = base + size - 1, 840 }; 841 842 /* decoders are enabled if committed */ 843 if (committed) { 844 cxld->flags |= CXL_DECODER_F_ENABLE; 845 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK) 846 cxld->flags |= CXL_DECODER_F_LOCK; 847 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl)) 848 cxld->target_type = CXL_DECODER_HOSTONLYMEM; 849 else 850 cxld->target_type = CXL_DECODER_DEVMEM; 851 if (cxld->id != port->commit_end + 1) { 852 dev_warn(&port->dev, 853 "decoder%d.%d: Committed out of order\n", 854 port->id, cxld->id); 855 return -ENXIO; 856 } 857 858 if (size == 0) { 859 dev_warn(&port->dev, 860 "decoder%d.%d: Committed with zero size\n", 861 port->id, cxld->id); 862 return -ENXIO; 863 } 864 port->commit_end = cxld->id; 865 } else { 866 if (cxled) { 867 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 868 struct cxl_dev_state *cxlds = cxlmd->cxlds; 869 870 /* 871 * Default by devtype until a device arrives that needs 872 * more precision. 873 */ 874 if (cxlds->type == CXL_DEVTYPE_CLASSMEM) 875 cxld->target_type = CXL_DECODER_HOSTONLYMEM; 876 else 877 cxld->target_type = CXL_DECODER_DEVMEM; 878 } else { 879 /* To be overridden by region type at commit time */ 880 cxld->target_type = CXL_DECODER_HOSTONLYMEM; 881 } 882 883 if (!FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl) && 884 cxld->target_type == CXL_DECODER_HOSTONLYMEM) { 885 ctrl |= CXL_HDM_DECODER0_CTRL_HOSTONLY; 886 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which)); 887 } 888 } 889 rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl), 890 &cxld->interleave_ways); 891 if (rc) { 892 dev_warn(&port->dev, 893 "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n", 894 port->id, cxld->id, ctrl); 895 return rc; 896 } 897 rc = eig_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl), 898 &cxld->interleave_granularity); 899 if (rc) 900 return rc; 901 902 dev_dbg(&port->dev, "decoder%d.%d: range: %#llx-%#llx iw: %d ig: %d\n", 903 port->id, cxld->id, cxld->hpa_range.start, cxld->hpa_range.end, 904 cxld->interleave_ways, cxld->interleave_granularity); 905 906 if (!cxled) { 907 lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which)); 908 hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which)); 909 target_list.value = (hi << 32) + lo; 910 for (i = 0; i < cxld->interleave_ways; i++) 911 target_map[i] = target_list.target_id[i]; 912 913 return 0; 914 } 915 916 if (!committed) 917 return 0; 918 919 dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder); 920 if (remainder) { 921 dev_err(&port->dev, 922 "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n", 923 port->id, cxld->id, size, cxld->interleave_ways); 924 return -ENXIO; 925 } 926 lo = readl(hdm + CXL_HDM_DECODER0_SKIP_LOW(which)); 927 hi = readl(hdm + CXL_HDM_DECODER0_SKIP_HIGH(which)); 928 skip = (hi << 32) + lo; 929 rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip); 930 if (rc) { 931 dev_err(&port->dev, 932 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)", 933 port->id, cxld->id, *dpa_base, 934 *dpa_base + dpa_size + skip - 1, rc); 935 return rc; 936 } 937 *dpa_base += dpa_size + skip; 938 939 cxled->state = CXL_DECODER_STATE_AUTO; 940 941 return 0; 942 } 943 944 static void cxl_settle_decoders(struct cxl_hdm *cxlhdm) 945 { 946 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 947 int committed, i; 948 u32 ctrl; 949 950 if (!hdm) 951 return; 952 953 /* 954 * Since the register resource was recently claimed via request_region() 955 * be careful about trusting the "not-committed" status until the commit 956 * timeout has elapsed. The commit timeout is 10ms (CXL 2.0 957 * 8.2.5.12.20), but double it to be tolerant of any clock skew between 958 * host and target. 959 */ 960 for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) { 961 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i)); 962 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED) 963 committed++; 964 } 965 966 /* ensure that future checks of committed can be trusted */ 967 if (committed != cxlhdm->decoder_count) 968 msleep(20); 969 } 970 971 /** 972 * devm_cxl_enumerate_decoders - add decoder objects per HDM register set 973 * @cxlhdm: Structure to populate with HDM capabilities 974 * @info: cached DVSEC range register info 975 */ 976 int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm, 977 struct cxl_endpoint_dvsec_info *info) 978 { 979 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 980 struct cxl_port *port = cxlhdm->port; 981 int i; 982 u64 dpa_base = 0; 983 984 cxl_settle_decoders(cxlhdm); 985 986 for (i = 0; i < cxlhdm->decoder_count; i++) { 987 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 }; 988 int rc, target_count = cxlhdm->target_count; 989 struct cxl_decoder *cxld; 990 991 if (is_cxl_endpoint(port)) { 992 struct cxl_endpoint_decoder *cxled; 993 994 cxled = cxl_endpoint_decoder_alloc(port); 995 if (IS_ERR(cxled)) { 996 dev_warn(&port->dev, 997 "Failed to allocate decoder%d.%d\n", 998 port->id, i); 999 return PTR_ERR(cxled); 1000 } 1001 cxld = &cxled->cxld; 1002 } else { 1003 struct cxl_switch_decoder *cxlsd; 1004 1005 cxlsd = cxl_switch_decoder_alloc(port, target_count); 1006 if (IS_ERR(cxlsd)) { 1007 dev_warn(&port->dev, 1008 "Failed to allocate decoder%d.%d\n", 1009 port->id, i); 1010 return PTR_ERR(cxlsd); 1011 } 1012 cxld = &cxlsd->cxld; 1013 } 1014 1015 rc = init_hdm_decoder(port, cxld, target_map, hdm, i, 1016 &dpa_base, info); 1017 if (rc) { 1018 dev_warn(&port->dev, 1019 "Failed to initialize decoder%d.%d\n", 1020 port->id, i); 1021 put_device(&cxld->dev); 1022 return rc; 1023 } 1024 rc = add_hdm_decoder(port, cxld, target_map); 1025 if (rc) { 1026 dev_warn(&port->dev, 1027 "Failed to add decoder%d.%d\n", port->id, i); 1028 return rc; 1029 } 1030 } 1031 1032 return 0; 1033 } 1034 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, CXL); 1035