1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2021 Intel Corporation. All rights reserved. */ 3 #include <linux/io-64-nonatomic-lo-hi.h> 4 #include <linux/device.h> 5 #include <linux/delay.h> 6 #include <linux/pci.h> 7 #include <linux/pci-doe.h> 8 #include <cxlpci.h> 9 #include <cxlmem.h> 10 #include <cxl.h> 11 #include "core.h" 12 #include "trace.h" 13 14 /** 15 * DOC: cxl core pci 16 * 17 * Compute Express Link protocols are layered on top of PCIe. CXL core provides 18 * a set of helpers for CXL interactions which occur via PCIe. 19 */ 20 21 static unsigned short media_ready_timeout = 60; 22 module_param(media_ready_timeout, ushort, 0644); 23 MODULE_PARM_DESC(media_ready_timeout, "seconds to wait for media ready"); 24 25 struct cxl_walk_context { 26 struct pci_bus *bus; 27 struct cxl_port *port; 28 int type; 29 int error; 30 int count; 31 }; 32 match_add_dports(struct pci_dev * pdev,void * data)33 static int match_add_dports(struct pci_dev *pdev, void *data) 34 { 35 struct cxl_walk_context *ctx = data; 36 struct cxl_port *port = ctx->port; 37 int type = pci_pcie_type(pdev); 38 struct cxl_register_map map; 39 struct cxl_dport *dport; 40 u32 lnkcap, port_num; 41 int rc; 42 43 if (pdev->bus != ctx->bus) 44 return 0; 45 if (!pci_is_pcie(pdev)) 46 return 0; 47 if (type != ctx->type) 48 return 0; 49 if (pci_read_config_dword(pdev, pci_pcie_cap(pdev) + PCI_EXP_LNKCAP, 50 &lnkcap)) 51 return 0; 52 53 rc = cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map); 54 if (rc) 55 dev_dbg(&port->dev, "failed to find component registers\n"); 56 57 port_num = FIELD_GET(PCI_EXP_LNKCAP_PN, lnkcap); 58 dport = devm_cxl_add_dport(port, &pdev->dev, port_num, map.resource); 59 if (IS_ERR(dport)) { 60 ctx->error = PTR_ERR(dport); 61 return PTR_ERR(dport); 62 } 63 ctx->count++; 64 65 return 0; 66 } 67 68 /** 69 * devm_cxl_port_enumerate_dports - enumerate downstream ports of the upstream port 70 * @port: cxl_port whose ->uport_dev is the upstream of dports to be enumerated 71 * 72 * Returns a positive number of dports enumerated or a negative error 73 * code. 74 */ devm_cxl_port_enumerate_dports(struct cxl_port * port)75 int devm_cxl_port_enumerate_dports(struct cxl_port *port) 76 { 77 struct pci_bus *bus = cxl_port_to_pci_bus(port); 78 struct cxl_walk_context ctx; 79 int type; 80 81 if (!bus) 82 return -ENXIO; 83 84 if (pci_is_root_bus(bus)) 85 type = PCI_EXP_TYPE_ROOT_PORT; 86 else 87 type = PCI_EXP_TYPE_DOWNSTREAM; 88 89 ctx = (struct cxl_walk_context) { 90 .port = port, 91 .bus = bus, 92 .type = type, 93 }; 94 pci_walk_bus(bus, match_add_dports, &ctx); 95 96 if (ctx.count == 0) 97 return -ENODEV; 98 if (ctx.error) 99 return ctx.error; 100 return ctx.count; 101 } 102 EXPORT_SYMBOL_NS_GPL(devm_cxl_port_enumerate_dports, CXL); 103 cxl_dvsec_mem_range_valid(struct cxl_dev_state * cxlds,int id)104 static int cxl_dvsec_mem_range_valid(struct cxl_dev_state *cxlds, int id) 105 { 106 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 107 int d = cxlds->cxl_dvsec; 108 bool valid = false; 109 int rc, i; 110 u32 temp; 111 112 if (id > CXL_DVSEC_RANGE_MAX) 113 return -EINVAL; 114 115 /* Check MEM INFO VALID bit first, give up after 1s */ 116 i = 1; 117 do { 118 rc = pci_read_config_dword(pdev, 119 d + CXL_DVSEC_RANGE_SIZE_LOW(id), 120 &temp); 121 if (rc) 122 return rc; 123 124 valid = FIELD_GET(CXL_DVSEC_MEM_INFO_VALID, temp); 125 if (valid) 126 break; 127 msleep(1000); 128 } while (i--); 129 130 if (!valid) { 131 dev_err(&pdev->dev, 132 "Timeout awaiting memory range %d valid after 1s.\n", 133 id); 134 return -ETIMEDOUT; 135 } 136 137 return 0; 138 } 139 cxl_dvsec_mem_range_active(struct cxl_dev_state * cxlds,int id)140 static int cxl_dvsec_mem_range_active(struct cxl_dev_state *cxlds, int id) 141 { 142 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 143 int d = cxlds->cxl_dvsec; 144 bool active = false; 145 int rc, i; 146 u32 temp; 147 148 if (id > CXL_DVSEC_RANGE_MAX) 149 return -EINVAL; 150 151 /* Check MEM ACTIVE bit, up to 60s timeout by default */ 152 for (i = media_ready_timeout; i; i--) { 153 rc = pci_read_config_dword( 154 pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(id), &temp); 155 if (rc) 156 return rc; 157 158 active = FIELD_GET(CXL_DVSEC_MEM_ACTIVE, temp); 159 if (active) 160 break; 161 msleep(1000); 162 } 163 164 if (!active) { 165 dev_err(&pdev->dev, 166 "timeout awaiting memory active after %d seconds\n", 167 media_ready_timeout); 168 return -ETIMEDOUT; 169 } 170 171 return 0; 172 } 173 174 /* 175 * Wait up to @media_ready_timeout for the device to report memory 176 * active. 177 */ cxl_await_media_ready(struct cxl_dev_state * cxlds)178 int cxl_await_media_ready(struct cxl_dev_state *cxlds) 179 { 180 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 181 int d = cxlds->cxl_dvsec; 182 int rc, i, hdm_count; 183 u64 md_status; 184 u16 cap; 185 186 rc = pci_read_config_word(pdev, 187 d + CXL_DVSEC_CAP_OFFSET, &cap); 188 if (rc) 189 return rc; 190 191 hdm_count = FIELD_GET(CXL_DVSEC_HDM_COUNT_MASK, cap); 192 for (i = 0; i < hdm_count; i++) { 193 rc = cxl_dvsec_mem_range_valid(cxlds, i); 194 if (rc) 195 return rc; 196 } 197 198 for (i = 0; i < hdm_count; i++) { 199 rc = cxl_dvsec_mem_range_active(cxlds, i); 200 if (rc) 201 return rc; 202 } 203 204 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); 205 if (!CXLMDEV_READY(md_status)) 206 return -EIO; 207 208 return 0; 209 } 210 EXPORT_SYMBOL_NS_GPL(cxl_await_media_ready, CXL); 211 wait_for_valid(struct pci_dev * pdev,int d)212 static int wait_for_valid(struct pci_dev *pdev, int d) 213 { 214 u32 val; 215 int rc; 216 217 /* 218 * Memory_Info_Valid: When set, indicates that the CXL Range 1 Size high 219 * and Size Low registers are valid. Must be set within 1 second of 220 * deassertion of reset to CXL device. Likely it is already set by the 221 * time this runs, but otherwise give a 1.5 second timeout in case of 222 * clock skew. 223 */ 224 rc = pci_read_config_dword(pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &val); 225 if (rc) 226 return rc; 227 228 if (val & CXL_DVSEC_MEM_INFO_VALID) 229 return 0; 230 231 msleep(1500); 232 233 rc = pci_read_config_dword(pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &val); 234 if (rc) 235 return rc; 236 237 if (val & CXL_DVSEC_MEM_INFO_VALID) 238 return 0; 239 240 return -ETIMEDOUT; 241 } 242 cxl_set_mem_enable(struct cxl_dev_state * cxlds,u16 val)243 static int cxl_set_mem_enable(struct cxl_dev_state *cxlds, u16 val) 244 { 245 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 246 int d = cxlds->cxl_dvsec; 247 u16 ctrl; 248 int rc; 249 250 rc = pci_read_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, &ctrl); 251 if (rc < 0) 252 return rc; 253 254 if ((ctrl & CXL_DVSEC_MEM_ENABLE) == val) 255 return 1; 256 ctrl &= ~CXL_DVSEC_MEM_ENABLE; 257 ctrl |= val; 258 259 rc = pci_write_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, ctrl); 260 if (rc < 0) 261 return rc; 262 263 return 0; 264 } 265 clear_mem_enable(void * cxlds)266 static void clear_mem_enable(void *cxlds) 267 { 268 cxl_set_mem_enable(cxlds, 0); 269 } 270 devm_cxl_enable_mem(struct device * host,struct cxl_dev_state * cxlds)271 static int devm_cxl_enable_mem(struct device *host, struct cxl_dev_state *cxlds) 272 { 273 int rc; 274 275 rc = cxl_set_mem_enable(cxlds, CXL_DVSEC_MEM_ENABLE); 276 if (rc < 0) 277 return rc; 278 if (rc > 0) 279 return 0; 280 return devm_add_action_or_reset(host, clear_mem_enable, cxlds); 281 } 282 283 /* require dvsec ranges to be covered by a locked platform window */ dvsec_range_allowed(struct device * dev,void * arg)284 static int dvsec_range_allowed(struct device *dev, void *arg) 285 { 286 struct range *dev_range = arg; 287 struct cxl_decoder *cxld; 288 289 if (!is_root_decoder(dev)) 290 return 0; 291 292 cxld = to_cxl_decoder(dev); 293 294 if (!(cxld->flags & CXL_DECODER_F_RAM)) 295 return 0; 296 297 return range_contains(&cxld->hpa_range, dev_range); 298 } 299 disable_hdm(void * _cxlhdm)300 static void disable_hdm(void *_cxlhdm) 301 { 302 u32 global_ctrl; 303 struct cxl_hdm *cxlhdm = _cxlhdm; 304 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 305 306 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET); 307 writel(global_ctrl & ~CXL_HDM_DECODER_ENABLE, 308 hdm + CXL_HDM_DECODER_CTRL_OFFSET); 309 } 310 devm_cxl_enable_hdm(struct device * host,struct cxl_hdm * cxlhdm)311 static int devm_cxl_enable_hdm(struct device *host, struct cxl_hdm *cxlhdm) 312 { 313 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 314 u32 global_ctrl; 315 316 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET); 317 writel(global_ctrl | CXL_HDM_DECODER_ENABLE, 318 hdm + CXL_HDM_DECODER_CTRL_OFFSET); 319 320 return devm_add_action_or_reset(host, disable_hdm, cxlhdm); 321 } 322 cxl_dvsec_rr_decode(struct device * dev,int d,struct cxl_endpoint_dvsec_info * info)323 int cxl_dvsec_rr_decode(struct device *dev, int d, 324 struct cxl_endpoint_dvsec_info *info) 325 { 326 struct pci_dev *pdev = to_pci_dev(dev); 327 int hdm_count, rc, i, ranges = 0; 328 u16 cap, ctrl; 329 330 if (!d) { 331 dev_dbg(dev, "No DVSEC Capability\n"); 332 return -ENXIO; 333 } 334 335 rc = pci_read_config_word(pdev, d + CXL_DVSEC_CAP_OFFSET, &cap); 336 if (rc) 337 return rc; 338 339 rc = pci_read_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, &ctrl); 340 if (rc) 341 return rc; 342 343 if (!(cap & CXL_DVSEC_MEM_CAPABLE)) { 344 dev_dbg(dev, "Not MEM Capable\n"); 345 return -ENXIO; 346 } 347 348 /* 349 * It is not allowed by spec for MEM.capable to be set and have 0 legacy 350 * HDM decoders (values > 2 are also undefined as of CXL 2.0). As this 351 * driver is for a spec defined class code which must be CXL.mem 352 * capable, there is no point in continuing to enable CXL.mem. 353 */ 354 hdm_count = FIELD_GET(CXL_DVSEC_HDM_COUNT_MASK, cap); 355 if (!hdm_count || hdm_count > 2) 356 return -EINVAL; 357 358 rc = wait_for_valid(pdev, d); 359 if (rc) { 360 dev_dbg(dev, "Failure awaiting MEM_INFO_VALID (%d)\n", rc); 361 return rc; 362 } 363 364 /* 365 * The current DVSEC values are moot if the memory capability is 366 * disabled, and they will remain moot after the HDM Decoder 367 * capability is enabled. 368 */ 369 info->mem_enabled = FIELD_GET(CXL_DVSEC_MEM_ENABLE, ctrl); 370 if (!info->mem_enabled) 371 return 0; 372 373 for (i = 0; i < hdm_count; i++) { 374 u64 base, size; 375 u32 temp; 376 377 rc = pci_read_config_dword( 378 pdev, d + CXL_DVSEC_RANGE_SIZE_HIGH(i), &temp); 379 if (rc) 380 return rc; 381 382 size = (u64)temp << 32; 383 384 rc = pci_read_config_dword( 385 pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(i), &temp); 386 if (rc) 387 return rc; 388 389 size |= temp & CXL_DVSEC_MEM_SIZE_LOW_MASK; 390 if (!size) { 391 continue; 392 } 393 394 rc = pci_read_config_dword( 395 pdev, d + CXL_DVSEC_RANGE_BASE_HIGH(i), &temp); 396 if (rc) 397 return rc; 398 399 base = (u64)temp << 32; 400 401 rc = pci_read_config_dword( 402 pdev, d + CXL_DVSEC_RANGE_BASE_LOW(i), &temp); 403 if (rc) 404 return rc; 405 406 base |= temp & CXL_DVSEC_MEM_BASE_LOW_MASK; 407 408 info->dvsec_range[ranges++] = (struct range) { 409 .start = base, 410 .end = base + size - 1 411 }; 412 } 413 414 info->ranges = ranges; 415 416 return 0; 417 } 418 EXPORT_SYMBOL_NS_GPL(cxl_dvsec_rr_decode, CXL); 419 420 /** 421 * cxl_hdm_decode_init() - Setup HDM decoding for the endpoint 422 * @cxlds: Device state 423 * @cxlhdm: Mapped HDM decoder Capability 424 * @info: Cached DVSEC range registers info 425 * 426 * Try to enable the endpoint's HDM Decoder Capability 427 */ cxl_hdm_decode_init(struct cxl_dev_state * cxlds,struct cxl_hdm * cxlhdm,struct cxl_endpoint_dvsec_info * info)428 int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm, 429 struct cxl_endpoint_dvsec_info *info) 430 { 431 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 432 struct cxl_port *port = cxlhdm->port; 433 struct device *dev = cxlds->dev; 434 struct cxl_port *root; 435 int i, rc, allowed; 436 u32 global_ctrl = 0; 437 438 if (hdm) 439 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET); 440 441 /* 442 * If the HDM Decoder Capability is already enabled then assume 443 * that some other agent like platform firmware set it up. 444 */ 445 if (global_ctrl & CXL_HDM_DECODER_ENABLE || (!hdm && info->mem_enabled)) 446 return devm_cxl_enable_mem(&port->dev, cxlds); 447 else if (!hdm) 448 return -ENODEV; 449 450 root = to_cxl_port(port->dev.parent); 451 while (!is_cxl_root(root) && is_cxl_port(root->dev.parent)) 452 root = to_cxl_port(root->dev.parent); 453 if (!is_cxl_root(root)) { 454 dev_err(dev, "Failed to acquire root port for HDM enable\n"); 455 return -ENODEV; 456 } 457 458 for (i = 0, allowed = 0; info->mem_enabled && i < info->ranges; i++) { 459 struct device *cxld_dev; 460 461 cxld_dev = device_find_child(&root->dev, &info->dvsec_range[i], 462 dvsec_range_allowed); 463 if (!cxld_dev) { 464 dev_dbg(dev, "DVSEC Range%d denied by platform\n", i); 465 continue; 466 } 467 dev_dbg(dev, "DVSEC Range%d allowed by platform\n", i); 468 put_device(cxld_dev); 469 allowed++; 470 } 471 472 if (!allowed && info->mem_enabled) { 473 dev_err(dev, "Range register decodes outside platform defined CXL ranges.\n"); 474 return -ENXIO; 475 } 476 477 /* 478 * Per CXL 2.0 Section 8.1.3.8.3 and 8.1.3.8.4 DVSEC CXL Range 1 Base 479 * [High,Low] when HDM operation is enabled the range register values 480 * are ignored by the device, but the spec also recommends matching the 481 * DVSEC Range 1,2 to HDM Decoder Range 0,1. So, non-zero info->ranges 482 * are expected even though Linux does not require or maintain that 483 * match. If at least one DVSEC range is enabled and allowed, skip HDM 484 * Decoder Capability Enable. 485 */ 486 if (info->mem_enabled) 487 return 0; 488 489 rc = devm_cxl_enable_hdm(&port->dev, cxlhdm); 490 if (rc) 491 return rc; 492 493 return devm_cxl_enable_mem(&port->dev, cxlds); 494 } 495 EXPORT_SYMBOL_NS_GPL(cxl_hdm_decode_init, CXL); 496 497 #define CXL_DOE_TABLE_ACCESS_REQ_CODE 0x000000ff 498 #define CXL_DOE_TABLE_ACCESS_REQ_CODE_READ 0 499 #define CXL_DOE_TABLE_ACCESS_TABLE_TYPE 0x0000ff00 500 #define CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA 0 501 #define CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE 0xffff0000 502 #define CXL_DOE_TABLE_ACCESS_LAST_ENTRY 0xffff 503 #define CXL_DOE_PROTOCOL_TABLE_ACCESS 2 504 505 #define CDAT_DOE_REQ(entry_handle) cpu_to_le32 \ 506 (FIELD_PREP(CXL_DOE_TABLE_ACCESS_REQ_CODE, \ 507 CXL_DOE_TABLE_ACCESS_REQ_CODE_READ) | \ 508 FIELD_PREP(CXL_DOE_TABLE_ACCESS_TABLE_TYPE, \ 509 CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA) | \ 510 FIELD_PREP(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, (entry_handle))) 511 cxl_cdat_get_length(struct device * dev,struct pci_doe_mb * cdat_doe,size_t * length)512 static int cxl_cdat_get_length(struct device *dev, 513 struct pci_doe_mb *cdat_doe, 514 size_t *length) 515 { 516 __le32 request = CDAT_DOE_REQ(0); 517 __le32 response[2]; 518 int rc; 519 520 rc = pci_doe(cdat_doe, PCI_DVSEC_VENDOR_ID_CXL, 521 CXL_DOE_PROTOCOL_TABLE_ACCESS, 522 &request, sizeof(request), 523 &response, sizeof(response)); 524 if (rc < 0) { 525 dev_err(dev, "DOE failed: %d", rc); 526 return rc; 527 } 528 if (rc < sizeof(response)) 529 return -EIO; 530 531 *length = le32_to_cpu(response[1]); 532 dev_dbg(dev, "CDAT length %zu\n", *length); 533 534 return 0; 535 } 536 cxl_cdat_read_table(struct device * dev,struct pci_doe_mb * cdat_doe,void * cdat_table,size_t * cdat_length)537 static int cxl_cdat_read_table(struct device *dev, 538 struct pci_doe_mb *cdat_doe, 539 void *cdat_table, size_t *cdat_length) 540 { 541 size_t length = *cdat_length + sizeof(__le32); 542 __le32 *data = cdat_table; 543 int entry_handle = 0; 544 __le32 saved_dw = 0; 545 546 do { 547 __le32 request = CDAT_DOE_REQ(entry_handle); 548 struct cdat_entry_header *entry; 549 size_t entry_dw; 550 int rc; 551 552 rc = pci_doe(cdat_doe, PCI_DVSEC_VENDOR_ID_CXL, 553 CXL_DOE_PROTOCOL_TABLE_ACCESS, 554 &request, sizeof(request), 555 data, length); 556 if (rc < 0) { 557 dev_err(dev, "DOE failed: %d", rc); 558 return rc; 559 } 560 561 /* 1 DW Table Access Response Header + CDAT entry */ 562 entry = (struct cdat_entry_header *)(data + 1); 563 if ((entry_handle == 0 && 564 rc != sizeof(__le32) + sizeof(struct cdat_header)) || 565 (entry_handle > 0 && 566 (rc < sizeof(__le32) + sizeof(*entry) || 567 rc != sizeof(__le32) + le16_to_cpu(entry->length)))) 568 return -EIO; 569 570 /* Get the CXL table access header entry handle */ 571 entry_handle = FIELD_GET(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, 572 le32_to_cpu(data[0])); 573 entry_dw = rc / sizeof(__le32); 574 /* Skip Header */ 575 entry_dw -= 1; 576 /* 577 * Table Access Response Header overwrote the last DW of 578 * previous entry, so restore that DW 579 */ 580 *data = saved_dw; 581 length -= entry_dw * sizeof(__le32); 582 data += entry_dw; 583 saved_dw = *data; 584 } while (entry_handle != CXL_DOE_TABLE_ACCESS_LAST_ENTRY); 585 586 /* Length in CDAT header may exceed concatenation of CDAT entries */ 587 *cdat_length -= length - sizeof(__le32); 588 589 return 0; 590 } 591 592 /** 593 * read_cdat_data - Read the CDAT data on this port 594 * @port: Port to read data from 595 * 596 * This call will sleep waiting for responses from the DOE mailbox. 597 */ read_cdat_data(struct cxl_port * port)598 void read_cdat_data(struct cxl_port *port) 599 { 600 struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev); 601 struct device *host = cxlmd->dev.parent; 602 struct device *dev = &port->dev; 603 struct pci_doe_mb *cdat_doe; 604 size_t cdat_length; 605 void *cdat_table; 606 int rc; 607 608 if (!dev_is_pci(host)) 609 return; 610 cdat_doe = pci_find_doe_mailbox(to_pci_dev(host), 611 PCI_DVSEC_VENDOR_ID_CXL, 612 CXL_DOE_PROTOCOL_TABLE_ACCESS); 613 if (!cdat_doe) { 614 dev_dbg(dev, "No CDAT mailbox\n"); 615 return; 616 } 617 618 port->cdat_available = true; 619 620 if (cxl_cdat_get_length(dev, cdat_doe, &cdat_length)) { 621 dev_dbg(dev, "No CDAT length\n"); 622 return; 623 } 624 625 cdat_table = devm_kzalloc(dev, cdat_length + sizeof(__le32), 626 GFP_KERNEL); 627 if (!cdat_table) 628 return; 629 630 rc = cxl_cdat_read_table(dev, cdat_doe, cdat_table, &cdat_length); 631 if (rc) { 632 /* Don't leave table data allocated on error */ 633 devm_kfree(dev, cdat_table); 634 dev_err(dev, "CDAT data read error\n"); 635 return; 636 } 637 638 port->cdat.table = cdat_table + sizeof(__le32); 639 port->cdat.length = cdat_length; 640 } 641 EXPORT_SYMBOL_NS_GPL(read_cdat_data, CXL); 642 cxl_cor_error_detected(struct pci_dev * pdev)643 void cxl_cor_error_detected(struct pci_dev *pdev) 644 { 645 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); 646 void __iomem *addr; 647 u32 status; 648 649 if (!cxlds->regs.ras) 650 return; 651 652 addr = cxlds->regs.ras + CXL_RAS_CORRECTABLE_STATUS_OFFSET; 653 status = readl(addr); 654 if (status & CXL_RAS_CORRECTABLE_STATUS_MASK) { 655 writel(status & CXL_RAS_CORRECTABLE_STATUS_MASK, addr); 656 trace_cxl_aer_correctable_error(cxlds->cxlmd, status); 657 } 658 } 659 EXPORT_SYMBOL_NS_GPL(cxl_cor_error_detected, CXL); 660 661 /* CXL spec rev3.0 8.2.4.16.1 */ header_log_copy(struct cxl_dev_state * cxlds,u32 * log)662 static void header_log_copy(struct cxl_dev_state *cxlds, u32 *log) 663 { 664 void __iomem *addr; 665 u32 *log_addr; 666 int i, log_u32_size = CXL_HEADERLOG_SIZE / sizeof(u32); 667 668 addr = cxlds->regs.ras + CXL_RAS_HEADER_LOG_OFFSET; 669 log_addr = log; 670 671 for (i = 0; i < log_u32_size; i++) { 672 *log_addr = readl(addr); 673 log_addr++; 674 addr += sizeof(u32); 675 } 676 } 677 678 /* 679 * Log the state of the RAS status registers and prepare them to log the 680 * next error status. Return 1 if reset needed. 681 */ cxl_report_and_clear(struct cxl_dev_state * cxlds)682 static bool cxl_report_and_clear(struct cxl_dev_state *cxlds) 683 { 684 u32 hl[CXL_HEADERLOG_SIZE_U32]; 685 void __iomem *addr; 686 u32 status; 687 u32 fe; 688 689 if (!cxlds->regs.ras) 690 return false; 691 692 addr = cxlds->regs.ras + CXL_RAS_UNCORRECTABLE_STATUS_OFFSET; 693 status = readl(addr); 694 if (!(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK)) 695 return false; 696 697 /* If multiple errors, log header points to first error from ctrl reg */ 698 if (hweight32(status) > 1) { 699 void __iomem *rcc_addr = 700 cxlds->regs.ras + CXL_RAS_CAP_CONTROL_OFFSET; 701 702 fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK, 703 readl(rcc_addr))); 704 } else { 705 fe = status; 706 } 707 708 header_log_copy(cxlds, hl); 709 trace_cxl_aer_uncorrectable_error(cxlds->cxlmd, status, fe, hl); 710 writel(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK, addr); 711 712 return true; 713 } 714 cxl_error_detected(struct pci_dev * pdev,pci_channel_state_t state)715 pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, 716 pci_channel_state_t state) 717 { 718 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); 719 struct cxl_memdev *cxlmd = cxlds->cxlmd; 720 struct device *dev = &cxlmd->dev; 721 bool ue; 722 723 /* 724 * A frozen channel indicates an impending reset which is fatal to 725 * CXL.mem operation, and will likely crash the system. On the off 726 * chance the situation is recoverable dump the status of the RAS 727 * capability registers and bounce the active state of the memdev. 728 */ 729 ue = cxl_report_and_clear(cxlds); 730 731 switch (state) { 732 case pci_channel_io_normal: 733 if (ue) { 734 device_release_driver(dev); 735 return PCI_ERS_RESULT_NEED_RESET; 736 } 737 return PCI_ERS_RESULT_CAN_RECOVER; 738 case pci_channel_io_frozen: 739 dev_warn(&pdev->dev, 740 "%s: frozen state error detected, disable CXL.mem\n", 741 dev_name(dev)); 742 device_release_driver(dev); 743 return PCI_ERS_RESULT_NEED_RESET; 744 case pci_channel_io_perm_failure: 745 dev_warn(&pdev->dev, 746 "failure state error detected, request disconnect\n"); 747 return PCI_ERS_RESULT_DISCONNECT; 748 } 749 return PCI_ERS_RESULT_NEED_RESET; 750 } 751 EXPORT_SYMBOL_NS_GPL(cxl_error_detected, CXL); 752