1 /* 2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of version 2 of the GNU General Public License as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 */ 13 #include <linux/list_sort.h> 14 #include <linux/libnvdimm.h> 15 #include <linux/module.h> 16 #include <linux/mutex.h> 17 #include <linux/ndctl.h> 18 #include <linux/sysfs.h> 19 #include <linux/delay.h> 20 #include <linux/list.h> 21 #include <linux/acpi.h> 22 #include <linux/sort.h> 23 #include <linux/io.h> 24 #include <linux/nd.h> 25 #include <asm/cacheflush.h> 26 #include "nfit.h" 27 28 /* 29 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is 30 * irrelevant. 31 */ 32 #include <linux/io-64-nonatomic-hi-lo.h> 33 34 static bool force_enable_dimms; 35 module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR); 36 MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status"); 37 38 static unsigned int scrub_timeout = NFIT_ARS_TIMEOUT; 39 module_param(scrub_timeout, uint, S_IRUGO|S_IWUSR); 40 MODULE_PARM_DESC(scrub_timeout, "Initial scrub timeout in seconds"); 41 42 /* after three payloads of overflow, it's dead jim */ 43 static unsigned int scrub_overflow_abort = 3; 44 module_param(scrub_overflow_abort, uint, S_IRUGO|S_IWUSR); 45 MODULE_PARM_DESC(scrub_overflow_abort, 46 "Number of times we overflow ARS results before abort"); 47 48 static bool disable_vendor_specific; 49 module_param(disable_vendor_specific, bool, S_IRUGO); 50 MODULE_PARM_DESC(disable_vendor_specific, 51 "Limit commands to the publicly specified set"); 52 53 static unsigned long override_dsm_mask; 54 module_param(override_dsm_mask, ulong, S_IRUGO); 55 MODULE_PARM_DESC(override_dsm_mask, "Bitmask of allowed NVDIMM DSM functions"); 56 57 static int default_dsm_family = -1; 58 module_param(default_dsm_family, int, S_IRUGO); 59 MODULE_PARM_DESC(default_dsm_family, 60 "Try this DSM type first when identifying NVDIMM family"); 61 62 LIST_HEAD(acpi_descs); 63 DEFINE_MUTEX(acpi_desc_lock); 64 65 static struct workqueue_struct *nfit_wq; 66 67 struct nfit_table_prev { 68 struct list_head spas; 69 struct list_head memdevs; 70 struct list_head dcrs; 71 struct list_head bdws; 72 struct list_head idts; 73 struct list_head flushes; 74 }; 75 76 static guid_t nfit_uuid[NFIT_UUID_MAX]; 77 78 const guid_t *to_nfit_uuid(enum nfit_uuids id) 79 { 80 return &nfit_uuid[id]; 81 } 82 EXPORT_SYMBOL(to_nfit_uuid); 83 84 static struct acpi_nfit_desc *to_acpi_nfit_desc( 85 struct nvdimm_bus_descriptor *nd_desc) 86 { 87 return container_of(nd_desc, struct acpi_nfit_desc, nd_desc); 88 } 89 90 static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc) 91 { 92 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 93 94 /* 95 * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct 96 * acpi_device. 97 */ 98 if (!nd_desc->provider_name 99 || strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0) 100 return NULL; 101 102 return to_acpi_device(acpi_desc->dev); 103 } 104 105 static int xlat_bus_status(void *buf, unsigned int cmd, u32 status) 106 { 107 struct nd_cmd_clear_error *clear_err; 108 struct nd_cmd_ars_status *ars_status; 109 u16 flags; 110 111 switch (cmd) { 112 case ND_CMD_ARS_CAP: 113 if ((status & 0xffff) == NFIT_ARS_CAP_NONE) 114 return -ENOTTY; 115 116 /* Command failed */ 117 if (status & 0xffff) 118 return -EIO; 119 120 /* No supported scan types for this range */ 121 flags = ND_ARS_PERSISTENT | ND_ARS_VOLATILE; 122 if ((status >> 16 & flags) == 0) 123 return -ENOTTY; 124 return 0; 125 case ND_CMD_ARS_START: 126 /* ARS is in progress */ 127 if ((status & 0xffff) == NFIT_ARS_START_BUSY) 128 return -EBUSY; 129 130 /* Command failed */ 131 if (status & 0xffff) 132 return -EIO; 133 return 0; 134 case ND_CMD_ARS_STATUS: 135 ars_status = buf; 136 /* Command failed */ 137 if (status & 0xffff) 138 return -EIO; 139 /* Check extended status (Upper two bytes) */ 140 if (status == NFIT_ARS_STATUS_DONE) 141 return 0; 142 143 /* ARS is in progress */ 144 if (status == NFIT_ARS_STATUS_BUSY) 145 return -EBUSY; 146 147 /* No ARS performed for the current boot */ 148 if (status == NFIT_ARS_STATUS_NONE) 149 return -EAGAIN; 150 151 /* 152 * ARS interrupted, either we overflowed or some other 153 * agent wants the scan to stop. If we didn't overflow 154 * then just continue with the returned results. 155 */ 156 if (status == NFIT_ARS_STATUS_INTR) { 157 if (ars_status->out_length >= 40 && (ars_status->flags 158 & NFIT_ARS_F_OVERFLOW)) 159 return -ENOSPC; 160 return 0; 161 } 162 163 /* Unknown status */ 164 if (status >> 16) 165 return -EIO; 166 return 0; 167 case ND_CMD_CLEAR_ERROR: 168 clear_err = buf; 169 if (status & 0xffff) 170 return -EIO; 171 if (!clear_err->cleared) 172 return -EIO; 173 if (clear_err->length > clear_err->cleared) 174 return clear_err->cleared; 175 return 0; 176 default: 177 break; 178 } 179 180 /* all other non-zero status results in an error */ 181 if (status) 182 return -EIO; 183 return 0; 184 } 185 186 #define ACPI_LABELS_LOCKED 3 187 188 static int xlat_nvdimm_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd, 189 u32 status) 190 { 191 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 192 193 switch (cmd) { 194 case ND_CMD_GET_CONFIG_SIZE: 195 /* 196 * In the _LSI, _LSR, _LSW case the locked status is 197 * communicated via the read/write commands 198 */ 199 if (nfit_mem->has_lsi) 200 break; 201 202 if (status >> 16 & ND_CONFIG_LOCKED) 203 return -EACCES; 204 break; 205 case ND_CMD_GET_CONFIG_DATA: 206 if (nfit_mem->has_lsr && status == ACPI_LABELS_LOCKED) 207 return -EACCES; 208 break; 209 case ND_CMD_SET_CONFIG_DATA: 210 if (nfit_mem->has_lsw && status == ACPI_LABELS_LOCKED) 211 return -EACCES; 212 break; 213 default: 214 break; 215 } 216 217 /* all other non-zero status results in an error */ 218 if (status) 219 return -EIO; 220 return 0; 221 } 222 223 static int xlat_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd, 224 u32 status) 225 { 226 if (!nvdimm) 227 return xlat_bus_status(buf, cmd, status); 228 return xlat_nvdimm_status(nvdimm, buf, cmd, status); 229 } 230 231 /* convert _LS{I,R} packages to the buffer object acpi_nfit_ctl expects */ 232 static union acpi_object *pkg_to_buf(union acpi_object *pkg) 233 { 234 int i; 235 void *dst; 236 size_t size = 0; 237 union acpi_object *buf = NULL; 238 239 if (pkg->type != ACPI_TYPE_PACKAGE) { 240 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n", 241 pkg->type); 242 goto err; 243 } 244 245 for (i = 0; i < pkg->package.count; i++) { 246 union acpi_object *obj = &pkg->package.elements[i]; 247 248 if (obj->type == ACPI_TYPE_INTEGER) 249 size += 4; 250 else if (obj->type == ACPI_TYPE_BUFFER) 251 size += obj->buffer.length; 252 else { 253 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n", 254 obj->type); 255 goto err; 256 } 257 } 258 259 buf = ACPI_ALLOCATE(sizeof(*buf) + size); 260 if (!buf) 261 goto err; 262 263 dst = buf + 1; 264 buf->type = ACPI_TYPE_BUFFER; 265 buf->buffer.length = size; 266 buf->buffer.pointer = dst; 267 for (i = 0; i < pkg->package.count; i++) { 268 union acpi_object *obj = &pkg->package.elements[i]; 269 270 if (obj->type == ACPI_TYPE_INTEGER) { 271 memcpy(dst, &obj->integer.value, 4); 272 dst += 4; 273 } else if (obj->type == ACPI_TYPE_BUFFER) { 274 memcpy(dst, obj->buffer.pointer, obj->buffer.length); 275 dst += obj->buffer.length; 276 } 277 } 278 err: 279 ACPI_FREE(pkg); 280 return buf; 281 } 282 283 static union acpi_object *int_to_buf(union acpi_object *integer) 284 { 285 union acpi_object *buf = ACPI_ALLOCATE(sizeof(*buf) + 4); 286 void *dst = NULL; 287 288 if (!buf) 289 goto err; 290 291 if (integer->type != ACPI_TYPE_INTEGER) { 292 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n", 293 integer->type); 294 goto err; 295 } 296 297 dst = buf + 1; 298 buf->type = ACPI_TYPE_BUFFER; 299 buf->buffer.length = 4; 300 buf->buffer.pointer = dst; 301 memcpy(dst, &integer->integer.value, 4); 302 err: 303 ACPI_FREE(integer); 304 return buf; 305 } 306 307 static union acpi_object *acpi_label_write(acpi_handle handle, u32 offset, 308 u32 len, void *data) 309 { 310 acpi_status rc; 311 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 312 struct acpi_object_list input = { 313 .count = 3, 314 .pointer = (union acpi_object []) { 315 [0] = { 316 .integer.type = ACPI_TYPE_INTEGER, 317 .integer.value = offset, 318 }, 319 [1] = { 320 .integer.type = ACPI_TYPE_INTEGER, 321 .integer.value = len, 322 }, 323 [2] = { 324 .buffer.type = ACPI_TYPE_BUFFER, 325 .buffer.pointer = data, 326 .buffer.length = len, 327 }, 328 }, 329 }; 330 331 rc = acpi_evaluate_object(handle, "_LSW", &input, &buf); 332 if (ACPI_FAILURE(rc)) 333 return NULL; 334 return int_to_buf(buf.pointer); 335 } 336 337 static union acpi_object *acpi_label_read(acpi_handle handle, u32 offset, 338 u32 len) 339 { 340 acpi_status rc; 341 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 342 struct acpi_object_list input = { 343 .count = 2, 344 .pointer = (union acpi_object []) { 345 [0] = { 346 .integer.type = ACPI_TYPE_INTEGER, 347 .integer.value = offset, 348 }, 349 [1] = { 350 .integer.type = ACPI_TYPE_INTEGER, 351 .integer.value = len, 352 }, 353 }, 354 }; 355 356 rc = acpi_evaluate_object(handle, "_LSR", &input, &buf); 357 if (ACPI_FAILURE(rc)) 358 return NULL; 359 return pkg_to_buf(buf.pointer); 360 } 361 362 static union acpi_object *acpi_label_info(acpi_handle handle) 363 { 364 acpi_status rc; 365 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 366 367 rc = acpi_evaluate_object(handle, "_LSI", NULL, &buf); 368 if (ACPI_FAILURE(rc)) 369 return NULL; 370 return pkg_to_buf(buf.pointer); 371 } 372 373 static u8 nfit_dsm_revid(unsigned family, unsigned func) 374 { 375 static const u8 revid_table[NVDIMM_FAMILY_MAX+1][32] = { 376 [NVDIMM_FAMILY_INTEL] = { 377 [NVDIMM_INTEL_GET_MODES] = 2, 378 [NVDIMM_INTEL_GET_FWINFO] = 2, 379 [NVDIMM_INTEL_START_FWUPDATE] = 2, 380 [NVDIMM_INTEL_SEND_FWUPDATE] = 2, 381 [NVDIMM_INTEL_FINISH_FWUPDATE] = 2, 382 [NVDIMM_INTEL_QUERY_FWUPDATE] = 2, 383 [NVDIMM_INTEL_SET_THRESHOLD] = 2, 384 [NVDIMM_INTEL_INJECT_ERROR] = 2, 385 }, 386 }; 387 u8 id; 388 389 if (family > NVDIMM_FAMILY_MAX) 390 return 0; 391 if (func > 31) 392 return 0; 393 id = revid_table[family][func]; 394 if (id == 0) 395 return 1; /* default */ 396 return id; 397 } 398 399 int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm, 400 unsigned int cmd, void *buf, unsigned int buf_len, int *cmd_rc) 401 { 402 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 403 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 404 union acpi_object in_obj, in_buf, *out_obj; 405 const struct nd_cmd_desc *desc = NULL; 406 struct device *dev = acpi_desc->dev; 407 struct nd_cmd_pkg *call_pkg = NULL; 408 const char *cmd_name, *dimm_name; 409 unsigned long cmd_mask, dsm_mask; 410 u32 offset, fw_status = 0; 411 acpi_handle handle; 412 unsigned int func; 413 const guid_t *guid; 414 int rc, i; 415 416 func = cmd; 417 if (cmd == ND_CMD_CALL) { 418 call_pkg = buf; 419 func = call_pkg->nd_command; 420 421 for (i = 0; i < ARRAY_SIZE(call_pkg->nd_reserved2); i++) 422 if (call_pkg->nd_reserved2[i]) 423 return -EINVAL; 424 } 425 426 if (nvdimm) { 427 struct acpi_device *adev = nfit_mem->adev; 428 429 if (!adev) 430 return -ENOTTY; 431 if (call_pkg && nfit_mem->family != call_pkg->nd_family) 432 return -ENOTTY; 433 434 dimm_name = nvdimm_name(nvdimm); 435 cmd_name = nvdimm_cmd_name(cmd); 436 cmd_mask = nvdimm_cmd_mask(nvdimm); 437 dsm_mask = nfit_mem->dsm_mask; 438 desc = nd_cmd_dimm_desc(cmd); 439 guid = to_nfit_uuid(nfit_mem->family); 440 handle = adev->handle; 441 } else { 442 struct acpi_device *adev = to_acpi_dev(acpi_desc); 443 444 cmd_name = nvdimm_bus_cmd_name(cmd); 445 cmd_mask = nd_desc->cmd_mask; 446 dsm_mask = cmd_mask; 447 if (cmd == ND_CMD_CALL) 448 dsm_mask = nd_desc->bus_dsm_mask; 449 desc = nd_cmd_bus_desc(cmd); 450 guid = to_nfit_uuid(NFIT_DEV_BUS); 451 handle = adev->handle; 452 dimm_name = "bus"; 453 } 454 455 if (!desc || (cmd && (desc->out_num + desc->in_num == 0))) 456 return -ENOTTY; 457 458 if (!test_bit(cmd, &cmd_mask) || !test_bit(func, &dsm_mask)) 459 return -ENOTTY; 460 461 in_obj.type = ACPI_TYPE_PACKAGE; 462 in_obj.package.count = 1; 463 in_obj.package.elements = &in_buf; 464 in_buf.type = ACPI_TYPE_BUFFER; 465 in_buf.buffer.pointer = buf; 466 in_buf.buffer.length = 0; 467 468 /* libnvdimm has already validated the input envelope */ 469 for (i = 0; i < desc->in_num; i++) 470 in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc, 471 i, buf); 472 473 if (call_pkg) { 474 /* skip over package wrapper */ 475 in_buf.buffer.pointer = (void *) &call_pkg->nd_payload; 476 in_buf.buffer.length = call_pkg->nd_size_in; 477 } 478 479 dev_dbg(dev, "%s:%s cmd: %d: func: %d input length: %d\n", 480 __func__, dimm_name, cmd, func, in_buf.buffer.length); 481 print_hex_dump_debug("nvdimm in ", DUMP_PREFIX_OFFSET, 4, 4, 482 in_buf.buffer.pointer, 483 min_t(u32, 256, in_buf.buffer.length), true); 484 485 /* call the BIOS, prefer the named methods over _DSM if available */ 486 if (nvdimm && cmd == ND_CMD_GET_CONFIG_SIZE && nfit_mem->has_lsi) 487 out_obj = acpi_label_info(handle); 488 else if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && nfit_mem->has_lsr) { 489 struct nd_cmd_get_config_data_hdr *p = buf; 490 491 out_obj = acpi_label_read(handle, p->in_offset, p->in_length); 492 } else if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA 493 && nfit_mem->has_lsw) { 494 struct nd_cmd_set_config_hdr *p = buf; 495 496 out_obj = acpi_label_write(handle, p->in_offset, p->in_length, 497 p->in_buf); 498 } else { 499 u8 revid; 500 501 if (nvdimm) 502 revid = nfit_dsm_revid(nfit_mem->family, func); 503 else 504 revid = 1; 505 out_obj = acpi_evaluate_dsm(handle, guid, revid, func, &in_obj); 506 } 507 508 if (!out_obj) { 509 dev_dbg(dev, "%s:%s _DSM failed cmd: %s\n", __func__, dimm_name, 510 cmd_name); 511 return -EINVAL; 512 } 513 514 if (call_pkg) { 515 call_pkg->nd_fw_size = out_obj->buffer.length; 516 memcpy(call_pkg->nd_payload + call_pkg->nd_size_in, 517 out_obj->buffer.pointer, 518 min(call_pkg->nd_fw_size, call_pkg->nd_size_out)); 519 520 ACPI_FREE(out_obj); 521 /* 522 * Need to support FW function w/o known size in advance. 523 * Caller can determine required size based upon nd_fw_size. 524 * If we return an error (like elsewhere) then caller wouldn't 525 * be able to rely upon data returned to make calculation. 526 */ 527 return 0; 528 } 529 530 if (out_obj->package.type != ACPI_TYPE_BUFFER) { 531 dev_dbg(dev, "%s:%s unexpected output object type cmd: %s type: %d\n", 532 __func__, dimm_name, cmd_name, out_obj->type); 533 rc = -EINVAL; 534 goto out; 535 } 536 537 dev_dbg(dev, "%s:%s cmd: %s output length: %d\n", __func__, dimm_name, 538 cmd_name, out_obj->buffer.length); 539 print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4, 4, 540 out_obj->buffer.pointer, 541 min_t(u32, 128, out_obj->buffer.length), true); 542 543 for (i = 0, offset = 0; i < desc->out_num; i++) { 544 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf, 545 (u32 *) out_obj->buffer.pointer, 546 out_obj->buffer.length - offset); 547 548 if (offset + out_size > out_obj->buffer.length) { 549 dev_dbg(dev, "%s:%s output object underflow cmd: %s field: %d\n", 550 __func__, dimm_name, cmd_name, i); 551 break; 552 } 553 554 if (in_buf.buffer.length + offset + out_size > buf_len) { 555 dev_dbg(dev, "%s:%s output overrun cmd: %s field: %d\n", 556 __func__, dimm_name, cmd_name, i); 557 rc = -ENXIO; 558 goto out; 559 } 560 memcpy(buf + in_buf.buffer.length + offset, 561 out_obj->buffer.pointer + offset, out_size); 562 offset += out_size; 563 } 564 565 /* 566 * Set fw_status for all the commands with a known format to be 567 * later interpreted by xlat_status(). 568 */ 569 if (i >= 1 && ((!nvdimm && cmd >= ND_CMD_ARS_CAP 570 && cmd <= ND_CMD_CLEAR_ERROR) 571 || (nvdimm && cmd >= ND_CMD_SMART 572 && cmd <= ND_CMD_VENDOR))) 573 fw_status = *(u32 *) out_obj->buffer.pointer; 574 575 if (offset + in_buf.buffer.length < buf_len) { 576 if (i >= 1) { 577 /* 578 * status valid, return the number of bytes left 579 * unfilled in the output buffer 580 */ 581 rc = buf_len - offset - in_buf.buffer.length; 582 if (cmd_rc) 583 *cmd_rc = xlat_status(nvdimm, buf, cmd, 584 fw_status); 585 } else { 586 dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n", 587 __func__, dimm_name, cmd_name, buf_len, 588 offset); 589 rc = -ENXIO; 590 } 591 } else { 592 rc = 0; 593 if (cmd_rc) 594 *cmd_rc = xlat_status(nvdimm, buf, cmd, fw_status); 595 } 596 597 out: 598 ACPI_FREE(out_obj); 599 600 return rc; 601 } 602 EXPORT_SYMBOL_GPL(acpi_nfit_ctl); 603 604 static const char *spa_type_name(u16 type) 605 { 606 static const char *to_name[] = { 607 [NFIT_SPA_VOLATILE] = "volatile", 608 [NFIT_SPA_PM] = "pmem", 609 [NFIT_SPA_DCR] = "dimm-control-region", 610 [NFIT_SPA_BDW] = "block-data-window", 611 [NFIT_SPA_VDISK] = "volatile-disk", 612 [NFIT_SPA_VCD] = "volatile-cd", 613 [NFIT_SPA_PDISK] = "persistent-disk", 614 [NFIT_SPA_PCD] = "persistent-cd", 615 616 }; 617 618 if (type > NFIT_SPA_PCD) 619 return "unknown"; 620 621 return to_name[type]; 622 } 623 624 int nfit_spa_type(struct acpi_nfit_system_address *spa) 625 { 626 int i; 627 628 for (i = 0; i < NFIT_UUID_MAX; i++) 629 if (guid_equal(to_nfit_uuid(i), (guid_t *)&spa->range_guid)) 630 return i; 631 return -1; 632 } 633 634 static bool add_spa(struct acpi_nfit_desc *acpi_desc, 635 struct nfit_table_prev *prev, 636 struct acpi_nfit_system_address *spa) 637 { 638 struct device *dev = acpi_desc->dev; 639 struct nfit_spa *nfit_spa; 640 641 if (spa->header.length != sizeof(*spa)) 642 return false; 643 644 list_for_each_entry(nfit_spa, &prev->spas, list) { 645 if (memcmp(nfit_spa->spa, spa, sizeof(*spa)) == 0) { 646 list_move_tail(&nfit_spa->list, &acpi_desc->spas); 647 return true; 648 } 649 } 650 651 nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa) + sizeof(*spa), 652 GFP_KERNEL); 653 if (!nfit_spa) 654 return false; 655 INIT_LIST_HEAD(&nfit_spa->list); 656 memcpy(nfit_spa->spa, spa, sizeof(*spa)); 657 list_add_tail(&nfit_spa->list, &acpi_desc->spas); 658 dev_dbg(dev, "%s: spa index: %d type: %s\n", __func__, 659 spa->range_index, 660 spa_type_name(nfit_spa_type(spa))); 661 return true; 662 } 663 664 static bool add_memdev(struct acpi_nfit_desc *acpi_desc, 665 struct nfit_table_prev *prev, 666 struct acpi_nfit_memory_map *memdev) 667 { 668 struct device *dev = acpi_desc->dev; 669 struct nfit_memdev *nfit_memdev; 670 671 if (memdev->header.length != sizeof(*memdev)) 672 return false; 673 674 list_for_each_entry(nfit_memdev, &prev->memdevs, list) 675 if (memcmp(nfit_memdev->memdev, memdev, sizeof(*memdev)) == 0) { 676 list_move_tail(&nfit_memdev->list, &acpi_desc->memdevs); 677 return true; 678 } 679 680 nfit_memdev = devm_kzalloc(dev, sizeof(*nfit_memdev) + sizeof(*memdev), 681 GFP_KERNEL); 682 if (!nfit_memdev) 683 return false; 684 INIT_LIST_HEAD(&nfit_memdev->list); 685 memcpy(nfit_memdev->memdev, memdev, sizeof(*memdev)); 686 list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs); 687 dev_dbg(dev, "%s: memdev handle: %#x spa: %d dcr: %d flags: %#x\n", 688 __func__, memdev->device_handle, memdev->range_index, 689 memdev->region_index, memdev->flags); 690 return true; 691 } 692 693 /* 694 * An implementation may provide a truncated control region if no block windows 695 * are defined. 696 */ 697 static size_t sizeof_dcr(struct acpi_nfit_control_region *dcr) 698 { 699 if (dcr->header.length < offsetof(struct acpi_nfit_control_region, 700 window_size)) 701 return 0; 702 if (dcr->windows) 703 return sizeof(*dcr); 704 return offsetof(struct acpi_nfit_control_region, window_size); 705 } 706 707 static bool add_dcr(struct acpi_nfit_desc *acpi_desc, 708 struct nfit_table_prev *prev, 709 struct acpi_nfit_control_region *dcr) 710 { 711 struct device *dev = acpi_desc->dev; 712 struct nfit_dcr *nfit_dcr; 713 714 if (!sizeof_dcr(dcr)) 715 return false; 716 717 list_for_each_entry(nfit_dcr, &prev->dcrs, list) 718 if (memcmp(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)) == 0) { 719 list_move_tail(&nfit_dcr->list, &acpi_desc->dcrs); 720 return true; 721 } 722 723 nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr) + sizeof(*dcr), 724 GFP_KERNEL); 725 if (!nfit_dcr) 726 return false; 727 INIT_LIST_HEAD(&nfit_dcr->list); 728 memcpy(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)); 729 list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs); 730 dev_dbg(dev, "%s: dcr index: %d windows: %d\n", __func__, 731 dcr->region_index, dcr->windows); 732 return true; 733 } 734 735 static bool add_bdw(struct acpi_nfit_desc *acpi_desc, 736 struct nfit_table_prev *prev, 737 struct acpi_nfit_data_region *bdw) 738 { 739 struct device *dev = acpi_desc->dev; 740 struct nfit_bdw *nfit_bdw; 741 742 if (bdw->header.length != sizeof(*bdw)) 743 return false; 744 list_for_each_entry(nfit_bdw, &prev->bdws, list) 745 if (memcmp(nfit_bdw->bdw, bdw, sizeof(*bdw)) == 0) { 746 list_move_tail(&nfit_bdw->list, &acpi_desc->bdws); 747 return true; 748 } 749 750 nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw) + sizeof(*bdw), 751 GFP_KERNEL); 752 if (!nfit_bdw) 753 return false; 754 INIT_LIST_HEAD(&nfit_bdw->list); 755 memcpy(nfit_bdw->bdw, bdw, sizeof(*bdw)); 756 list_add_tail(&nfit_bdw->list, &acpi_desc->bdws); 757 dev_dbg(dev, "%s: bdw dcr: %d windows: %d\n", __func__, 758 bdw->region_index, bdw->windows); 759 return true; 760 } 761 762 static size_t sizeof_idt(struct acpi_nfit_interleave *idt) 763 { 764 if (idt->header.length < sizeof(*idt)) 765 return 0; 766 return sizeof(*idt) + sizeof(u32) * (idt->line_count - 1); 767 } 768 769 static bool add_idt(struct acpi_nfit_desc *acpi_desc, 770 struct nfit_table_prev *prev, 771 struct acpi_nfit_interleave *idt) 772 { 773 struct device *dev = acpi_desc->dev; 774 struct nfit_idt *nfit_idt; 775 776 if (!sizeof_idt(idt)) 777 return false; 778 779 list_for_each_entry(nfit_idt, &prev->idts, list) { 780 if (sizeof_idt(nfit_idt->idt) != sizeof_idt(idt)) 781 continue; 782 783 if (memcmp(nfit_idt->idt, idt, sizeof_idt(idt)) == 0) { 784 list_move_tail(&nfit_idt->list, &acpi_desc->idts); 785 return true; 786 } 787 } 788 789 nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt) + sizeof_idt(idt), 790 GFP_KERNEL); 791 if (!nfit_idt) 792 return false; 793 INIT_LIST_HEAD(&nfit_idt->list); 794 memcpy(nfit_idt->idt, idt, sizeof_idt(idt)); 795 list_add_tail(&nfit_idt->list, &acpi_desc->idts); 796 dev_dbg(dev, "%s: idt index: %d num_lines: %d\n", __func__, 797 idt->interleave_index, idt->line_count); 798 return true; 799 } 800 801 static size_t sizeof_flush(struct acpi_nfit_flush_address *flush) 802 { 803 if (flush->header.length < sizeof(*flush)) 804 return 0; 805 return sizeof(*flush) + sizeof(u64) * (flush->hint_count - 1); 806 } 807 808 static bool add_flush(struct acpi_nfit_desc *acpi_desc, 809 struct nfit_table_prev *prev, 810 struct acpi_nfit_flush_address *flush) 811 { 812 struct device *dev = acpi_desc->dev; 813 struct nfit_flush *nfit_flush; 814 815 if (!sizeof_flush(flush)) 816 return false; 817 818 list_for_each_entry(nfit_flush, &prev->flushes, list) { 819 if (sizeof_flush(nfit_flush->flush) != sizeof_flush(flush)) 820 continue; 821 822 if (memcmp(nfit_flush->flush, flush, 823 sizeof_flush(flush)) == 0) { 824 list_move_tail(&nfit_flush->list, &acpi_desc->flushes); 825 return true; 826 } 827 } 828 829 nfit_flush = devm_kzalloc(dev, sizeof(*nfit_flush) 830 + sizeof_flush(flush), GFP_KERNEL); 831 if (!nfit_flush) 832 return false; 833 INIT_LIST_HEAD(&nfit_flush->list); 834 memcpy(nfit_flush->flush, flush, sizeof_flush(flush)); 835 list_add_tail(&nfit_flush->list, &acpi_desc->flushes); 836 dev_dbg(dev, "%s: nfit_flush handle: %d hint_count: %d\n", __func__, 837 flush->device_handle, flush->hint_count); 838 return true; 839 } 840 841 static void *add_table(struct acpi_nfit_desc *acpi_desc, 842 struct nfit_table_prev *prev, void *table, const void *end) 843 { 844 struct device *dev = acpi_desc->dev; 845 struct acpi_nfit_header *hdr; 846 void *err = ERR_PTR(-ENOMEM); 847 848 if (table >= end) 849 return NULL; 850 851 hdr = table; 852 if (!hdr->length) { 853 dev_warn(dev, "found a zero length table '%d' parsing nfit\n", 854 hdr->type); 855 return NULL; 856 } 857 858 switch (hdr->type) { 859 case ACPI_NFIT_TYPE_SYSTEM_ADDRESS: 860 if (!add_spa(acpi_desc, prev, table)) 861 return err; 862 break; 863 case ACPI_NFIT_TYPE_MEMORY_MAP: 864 if (!add_memdev(acpi_desc, prev, table)) 865 return err; 866 break; 867 case ACPI_NFIT_TYPE_CONTROL_REGION: 868 if (!add_dcr(acpi_desc, prev, table)) 869 return err; 870 break; 871 case ACPI_NFIT_TYPE_DATA_REGION: 872 if (!add_bdw(acpi_desc, prev, table)) 873 return err; 874 break; 875 case ACPI_NFIT_TYPE_INTERLEAVE: 876 if (!add_idt(acpi_desc, prev, table)) 877 return err; 878 break; 879 case ACPI_NFIT_TYPE_FLUSH_ADDRESS: 880 if (!add_flush(acpi_desc, prev, table)) 881 return err; 882 break; 883 case ACPI_NFIT_TYPE_SMBIOS: 884 dev_dbg(dev, "%s: smbios\n", __func__); 885 break; 886 default: 887 dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type); 888 break; 889 } 890 891 return table + hdr->length; 892 } 893 894 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc, 895 struct nfit_mem *nfit_mem) 896 { 897 u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle; 898 u16 dcr = nfit_mem->dcr->region_index; 899 struct nfit_spa *nfit_spa; 900 901 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 902 u16 range_index = nfit_spa->spa->range_index; 903 int type = nfit_spa_type(nfit_spa->spa); 904 struct nfit_memdev *nfit_memdev; 905 906 if (type != NFIT_SPA_BDW) 907 continue; 908 909 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 910 if (nfit_memdev->memdev->range_index != range_index) 911 continue; 912 if (nfit_memdev->memdev->device_handle != device_handle) 913 continue; 914 if (nfit_memdev->memdev->region_index != dcr) 915 continue; 916 917 nfit_mem->spa_bdw = nfit_spa->spa; 918 return; 919 } 920 } 921 922 dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n", 923 nfit_mem->spa_dcr->range_index); 924 nfit_mem->bdw = NULL; 925 } 926 927 static void nfit_mem_init_bdw(struct acpi_nfit_desc *acpi_desc, 928 struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa) 929 { 930 u16 dcr = __to_nfit_memdev(nfit_mem)->region_index; 931 struct nfit_memdev *nfit_memdev; 932 struct nfit_bdw *nfit_bdw; 933 struct nfit_idt *nfit_idt; 934 u16 idt_idx, range_index; 935 936 list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) { 937 if (nfit_bdw->bdw->region_index != dcr) 938 continue; 939 nfit_mem->bdw = nfit_bdw->bdw; 940 break; 941 } 942 943 if (!nfit_mem->bdw) 944 return; 945 946 nfit_mem_find_spa_bdw(acpi_desc, nfit_mem); 947 948 if (!nfit_mem->spa_bdw) 949 return; 950 951 range_index = nfit_mem->spa_bdw->range_index; 952 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 953 if (nfit_memdev->memdev->range_index != range_index || 954 nfit_memdev->memdev->region_index != dcr) 955 continue; 956 nfit_mem->memdev_bdw = nfit_memdev->memdev; 957 idt_idx = nfit_memdev->memdev->interleave_index; 958 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) { 959 if (nfit_idt->idt->interleave_index != idt_idx) 960 continue; 961 nfit_mem->idt_bdw = nfit_idt->idt; 962 break; 963 } 964 break; 965 } 966 } 967 968 static int __nfit_mem_init(struct acpi_nfit_desc *acpi_desc, 969 struct acpi_nfit_system_address *spa) 970 { 971 struct nfit_mem *nfit_mem, *found; 972 struct nfit_memdev *nfit_memdev; 973 int type = spa ? nfit_spa_type(spa) : 0; 974 975 switch (type) { 976 case NFIT_SPA_DCR: 977 case NFIT_SPA_PM: 978 break; 979 default: 980 if (spa) 981 return 0; 982 } 983 984 /* 985 * This loop runs in two modes, when a dimm is mapped the loop 986 * adds memdev associations to an existing dimm, or creates a 987 * dimm. In the unmapped dimm case this loop sweeps for memdev 988 * instances with an invalid / zero range_index and adds those 989 * dimms without spa associations. 990 */ 991 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 992 struct nfit_flush *nfit_flush; 993 struct nfit_dcr *nfit_dcr; 994 u32 device_handle; 995 u16 dcr; 996 997 if (spa && nfit_memdev->memdev->range_index != spa->range_index) 998 continue; 999 if (!spa && nfit_memdev->memdev->range_index) 1000 continue; 1001 found = NULL; 1002 dcr = nfit_memdev->memdev->region_index; 1003 device_handle = nfit_memdev->memdev->device_handle; 1004 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) 1005 if (__to_nfit_memdev(nfit_mem)->device_handle 1006 == device_handle) { 1007 found = nfit_mem; 1008 break; 1009 } 1010 1011 if (found) 1012 nfit_mem = found; 1013 else { 1014 nfit_mem = devm_kzalloc(acpi_desc->dev, 1015 sizeof(*nfit_mem), GFP_KERNEL); 1016 if (!nfit_mem) 1017 return -ENOMEM; 1018 INIT_LIST_HEAD(&nfit_mem->list); 1019 nfit_mem->acpi_desc = acpi_desc; 1020 list_add(&nfit_mem->list, &acpi_desc->dimms); 1021 } 1022 1023 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) { 1024 if (nfit_dcr->dcr->region_index != dcr) 1025 continue; 1026 /* 1027 * Record the control region for the dimm. For 1028 * the ACPI 6.1 case, where there are separate 1029 * control regions for the pmem vs blk 1030 * interfaces, be sure to record the extended 1031 * blk details. 1032 */ 1033 if (!nfit_mem->dcr) 1034 nfit_mem->dcr = nfit_dcr->dcr; 1035 else if (nfit_mem->dcr->windows == 0 1036 && nfit_dcr->dcr->windows) 1037 nfit_mem->dcr = nfit_dcr->dcr; 1038 break; 1039 } 1040 1041 list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) { 1042 struct acpi_nfit_flush_address *flush; 1043 u16 i; 1044 1045 if (nfit_flush->flush->device_handle != device_handle) 1046 continue; 1047 nfit_mem->nfit_flush = nfit_flush; 1048 flush = nfit_flush->flush; 1049 nfit_mem->flush_wpq = devm_kzalloc(acpi_desc->dev, 1050 flush->hint_count 1051 * sizeof(struct resource), GFP_KERNEL); 1052 if (!nfit_mem->flush_wpq) 1053 return -ENOMEM; 1054 for (i = 0; i < flush->hint_count; i++) { 1055 struct resource *res = &nfit_mem->flush_wpq[i]; 1056 1057 res->start = flush->hint_address[i]; 1058 res->end = res->start + 8 - 1; 1059 } 1060 break; 1061 } 1062 1063 if (dcr && !nfit_mem->dcr) { 1064 dev_err(acpi_desc->dev, "SPA %d missing DCR %d\n", 1065 spa->range_index, dcr); 1066 return -ENODEV; 1067 } 1068 1069 if (type == NFIT_SPA_DCR) { 1070 struct nfit_idt *nfit_idt; 1071 u16 idt_idx; 1072 1073 /* multiple dimms may share a SPA when interleaved */ 1074 nfit_mem->spa_dcr = spa; 1075 nfit_mem->memdev_dcr = nfit_memdev->memdev; 1076 idt_idx = nfit_memdev->memdev->interleave_index; 1077 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) { 1078 if (nfit_idt->idt->interleave_index != idt_idx) 1079 continue; 1080 nfit_mem->idt_dcr = nfit_idt->idt; 1081 break; 1082 } 1083 nfit_mem_init_bdw(acpi_desc, nfit_mem, spa); 1084 } else if (type == NFIT_SPA_PM) { 1085 /* 1086 * A single dimm may belong to multiple SPA-PM 1087 * ranges, record at least one in addition to 1088 * any SPA-DCR range. 1089 */ 1090 nfit_mem->memdev_pmem = nfit_memdev->memdev; 1091 } else 1092 nfit_mem->memdev_dcr = nfit_memdev->memdev; 1093 } 1094 1095 return 0; 1096 } 1097 1098 static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b) 1099 { 1100 struct nfit_mem *a = container_of(_a, typeof(*a), list); 1101 struct nfit_mem *b = container_of(_b, typeof(*b), list); 1102 u32 handleA, handleB; 1103 1104 handleA = __to_nfit_memdev(a)->device_handle; 1105 handleB = __to_nfit_memdev(b)->device_handle; 1106 if (handleA < handleB) 1107 return -1; 1108 else if (handleA > handleB) 1109 return 1; 1110 return 0; 1111 } 1112 1113 static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc) 1114 { 1115 struct nfit_spa *nfit_spa; 1116 int rc; 1117 1118 1119 /* 1120 * For each SPA-DCR or SPA-PMEM address range find its 1121 * corresponding MEMDEV(s). From each MEMDEV find the 1122 * corresponding DCR. Then, if we're operating on a SPA-DCR, 1123 * try to find a SPA-BDW and a corresponding BDW that references 1124 * the DCR. Throw it all into an nfit_mem object. Note, that 1125 * BDWs are optional. 1126 */ 1127 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 1128 rc = __nfit_mem_init(acpi_desc, nfit_spa->spa); 1129 if (rc) 1130 return rc; 1131 } 1132 1133 /* 1134 * If a DIMM has failed to be mapped into SPA there will be no 1135 * SPA entries above. Find and register all the unmapped DIMMs 1136 * for reporting and recovery purposes. 1137 */ 1138 rc = __nfit_mem_init(acpi_desc, NULL); 1139 if (rc) 1140 return rc; 1141 1142 list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp); 1143 1144 return 0; 1145 } 1146 1147 static ssize_t bus_dsm_mask_show(struct device *dev, 1148 struct device_attribute *attr, char *buf) 1149 { 1150 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1151 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1152 1153 return sprintf(buf, "%#lx\n", nd_desc->bus_dsm_mask); 1154 } 1155 static struct device_attribute dev_attr_bus_dsm_mask = 1156 __ATTR(dsm_mask, 0444, bus_dsm_mask_show, NULL); 1157 1158 static ssize_t revision_show(struct device *dev, 1159 struct device_attribute *attr, char *buf) 1160 { 1161 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1162 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1163 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1164 1165 return sprintf(buf, "%d\n", acpi_desc->acpi_header.revision); 1166 } 1167 static DEVICE_ATTR_RO(revision); 1168 1169 static ssize_t hw_error_scrub_show(struct device *dev, 1170 struct device_attribute *attr, char *buf) 1171 { 1172 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1173 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1174 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1175 1176 return sprintf(buf, "%d\n", acpi_desc->scrub_mode); 1177 } 1178 1179 /* 1180 * The 'hw_error_scrub' attribute can have the following values written to it: 1181 * '0': Switch to the default mode where an exception will only insert 1182 * the address of the memory error into the poison and badblocks lists. 1183 * '1': Enable a full scrub to happen if an exception for a memory error is 1184 * received. 1185 */ 1186 static ssize_t hw_error_scrub_store(struct device *dev, 1187 struct device_attribute *attr, const char *buf, size_t size) 1188 { 1189 struct nvdimm_bus_descriptor *nd_desc; 1190 ssize_t rc; 1191 long val; 1192 1193 rc = kstrtol(buf, 0, &val); 1194 if (rc) 1195 return rc; 1196 1197 device_lock(dev); 1198 nd_desc = dev_get_drvdata(dev); 1199 if (nd_desc) { 1200 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1201 1202 switch (val) { 1203 case HW_ERROR_SCRUB_ON: 1204 acpi_desc->scrub_mode = HW_ERROR_SCRUB_ON; 1205 break; 1206 case HW_ERROR_SCRUB_OFF: 1207 acpi_desc->scrub_mode = HW_ERROR_SCRUB_OFF; 1208 break; 1209 default: 1210 rc = -EINVAL; 1211 break; 1212 } 1213 } 1214 device_unlock(dev); 1215 if (rc) 1216 return rc; 1217 return size; 1218 } 1219 static DEVICE_ATTR_RW(hw_error_scrub); 1220 1221 /* 1222 * This shows the number of full Address Range Scrubs that have been 1223 * completed since driver load time. Userspace can wait on this using 1224 * select/poll etc. A '+' at the end indicates an ARS is in progress 1225 */ 1226 static ssize_t scrub_show(struct device *dev, 1227 struct device_attribute *attr, char *buf) 1228 { 1229 struct nvdimm_bus_descriptor *nd_desc; 1230 ssize_t rc = -ENXIO; 1231 1232 device_lock(dev); 1233 nd_desc = dev_get_drvdata(dev); 1234 if (nd_desc) { 1235 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1236 1237 rc = sprintf(buf, "%d%s", acpi_desc->scrub_count, 1238 (work_busy(&acpi_desc->work)) ? "+\n" : "\n"); 1239 } 1240 device_unlock(dev); 1241 return rc; 1242 } 1243 1244 static ssize_t scrub_store(struct device *dev, 1245 struct device_attribute *attr, const char *buf, size_t size) 1246 { 1247 struct nvdimm_bus_descriptor *nd_desc; 1248 ssize_t rc; 1249 long val; 1250 1251 rc = kstrtol(buf, 0, &val); 1252 if (rc) 1253 return rc; 1254 if (val != 1) 1255 return -EINVAL; 1256 1257 device_lock(dev); 1258 nd_desc = dev_get_drvdata(dev); 1259 if (nd_desc) { 1260 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1261 1262 rc = acpi_nfit_ars_rescan(acpi_desc, 0); 1263 } 1264 device_unlock(dev); 1265 if (rc) 1266 return rc; 1267 return size; 1268 } 1269 static DEVICE_ATTR_RW(scrub); 1270 1271 static bool ars_supported(struct nvdimm_bus *nvdimm_bus) 1272 { 1273 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1274 const unsigned long mask = 1 << ND_CMD_ARS_CAP | 1 << ND_CMD_ARS_START 1275 | 1 << ND_CMD_ARS_STATUS; 1276 1277 return (nd_desc->cmd_mask & mask) == mask; 1278 } 1279 1280 static umode_t nfit_visible(struct kobject *kobj, struct attribute *a, int n) 1281 { 1282 struct device *dev = container_of(kobj, struct device, kobj); 1283 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1284 1285 if (a == &dev_attr_scrub.attr && !ars_supported(nvdimm_bus)) 1286 return 0; 1287 return a->mode; 1288 } 1289 1290 static struct attribute *acpi_nfit_attributes[] = { 1291 &dev_attr_revision.attr, 1292 &dev_attr_scrub.attr, 1293 &dev_attr_hw_error_scrub.attr, 1294 &dev_attr_bus_dsm_mask.attr, 1295 NULL, 1296 }; 1297 1298 static const struct attribute_group acpi_nfit_attribute_group = { 1299 .name = "nfit", 1300 .attrs = acpi_nfit_attributes, 1301 .is_visible = nfit_visible, 1302 }; 1303 1304 static const struct attribute_group *acpi_nfit_attribute_groups[] = { 1305 &nvdimm_bus_attribute_group, 1306 &acpi_nfit_attribute_group, 1307 NULL, 1308 }; 1309 1310 static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev) 1311 { 1312 struct nvdimm *nvdimm = to_nvdimm(dev); 1313 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1314 1315 return __to_nfit_memdev(nfit_mem); 1316 } 1317 1318 static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev) 1319 { 1320 struct nvdimm *nvdimm = to_nvdimm(dev); 1321 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1322 1323 return nfit_mem->dcr; 1324 } 1325 1326 static ssize_t handle_show(struct device *dev, 1327 struct device_attribute *attr, char *buf) 1328 { 1329 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev); 1330 1331 return sprintf(buf, "%#x\n", memdev->device_handle); 1332 } 1333 static DEVICE_ATTR_RO(handle); 1334 1335 static ssize_t phys_id_show(struct device *dev, 1336 struct device_attribute *attr, char *buf) 1337 { 1338 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev); 1339 1340 return sprintf(buf, "%#x\n", memdev->physical_id); 1341 } 1342 static DEVICE_ATTR_RO(phys_id); 1343 1344 static ssize_t vendor_show(struct device *dev, 1345 struct device_attribute *attr, char *buf) 1346 { 1347 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1348 1349 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->vendor_id)); 1350 } 1351 static DEVICE_ATTR_RO(vendor); 1352 1353 static ssize_t rev_id_show(struct device *dev, 1354 struct device_attribute *attr, char *buf) 1355 { 1356 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1357 1358 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->revision_id)); 1359 } 1360 static DEVICE_ATTR_RO(rev_id); 1361 1362 static ssize_t device_show(struct device *dev, 1363 struct device_attribute *attr, char *buf) 1364 { 1365 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1366 1367 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->device_id)); 1368 } 1369 static DEVICE_ATTR_RO(device); 1370 1371 static ssize_t subsystem_vendor_show(struct device *dev, 1372 struct device_attribute *attr, char *buf) 1373 { 1374 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1375 1376 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_vendor_id)); 1377 } 1378 static DEVICE_ATTR_RO(subsystem_vendor); 1379 1380 static ssize_t subsystem_rev_id_show(struct device *dev, 1381 struct device_attribute *attr, char *buf) 1382 { 1383 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1384 1385 return sprintf(buf, "0x%04x\n", 1386 be16_to_cpu(dcr->subsystem_revision_id)); 1387 } 1388 static DEVICE_ATTR_RO(subsystem_rev_id); 1389 1390 static ssize_t subsystem_device_show(struct device *dev, 1391 struct device_attribute *attr, char *buf) 1392 { 1393 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1394 1395 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_device_id)); 1396 } 1397 static DEVICE_ATTR_RO(subsystem_device); 1398 1399 static int num_nvdimm_formats(struct nvdimm *nvdimm) 1400 { 1401 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1402 int formats = 0; 1403 1404 if (nfit_mem->memdev_pmem) 1405 formats++; 1406 if (nfit_mem->memdev_bdw) 1407 formats++; 1408 return formats; 1409 } 1410 1411 static ssize_t format_show(struct device *dev, 1412 struct device_attribute *attr, char *buf) 1413 { 1414 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1415 1416 return sprintf(buf, "0x%04x\n", le16_to_cpu(dcr->code)); 1417 } 1418 static DEVICE_ATTR_RO(format); 1419 1420 static ssize_t format1_show(struct device *dev, 1421 struct device_attribute *attr, char *buf) 1422 { 1423 u32 handle; 1424 ssize_t rc = -ENXIO; 1425 struct nfit_mem *nfit_mem; 1426 struct nfit_memdev *nfit_memdev; 1427 struct acpi_nfit_desc *acpi_desc; 1428 struct nvdimm *nvdimm = to_nvdimm(dev); 1429 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1430 1431 nfit_mem = nvdimm_provider_data(nvdimm); 1432 acpi_desc = nfit_mem->acpi_desc; 1433 handle = to_nfit_memdev(dev)->device_handle; 1434 1435 /* assumes DIMMs have at most 2 published interface codes */ 1436 mutex_lock(&acpi_desc->init_mutex); 1437 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 1438 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev; 1439 struct nfit_dcr *nfit_dcr; 1440 1441 if (memdev->device_handle != handle) 1442 continue; 1443 1444 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) { 1445 if (nfit_dcr->dcr->region_index != memdev->region_index) 1446 continue; 1447 if (nfit_dcr->dcr->code == dcr->code) 1448 continue; 1449 rc = sprintf(buf, "0x%04x\n", 1450 le16_to_cpu(nfit_dcr->dcr->code)); 1451 break; 1452 } 1453 if (rc != ENXIO) 1454 break; 1455 } 1456 mutex_unlock(&acpi_desc->init_mutex); 1457 return rc; 1458 } 1459 static DEVICE_ATTR_RO(format1); 1460 1461 static ssize_t formats_show(struct device *dev, 1462 struct device_attribute *attr, char *buf) 1463 { 1464 struct nvdimm *nvdimm = to_nvdimm(dev); 1465 1466 return sprintf(buf, "%d\n", num_nvdimm_formats(nvdimm)); 1467 } 1468 static DEVICE_ATTR_RO(formats); 1469 1470 static ssize_t serial_show(struct device *dev, 1471 struct device_attribute *attr, char *buf) 1472 { 1473 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1474 1475 return sprintf(buf, "0x%08x\n", be32_to_cpu(dcr->serial_number)); 1476 } 1477 static DEVICE_ATTR_RO(serial); 1478 1479 static ssize_t family_show(struct device *dev, 1480 struct device_attribute *attr, char *buf) 1481 { 1482 struct nvdimm *nvdimm = to_nvdimm(dev); 1483 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1484 1485 if (nfit_mem->family < 0) 1486 return -ENXIO; 1487 return sprintf(buf, "%d\n", nfit_mem->family); 1488 } 1489 static DEVICE_ATTR_RO(family); 1490 1491 static ssize_t dsm_mask_show(struct device *dev, 1492 struct device_attribute *attr, char *buf) 1493 { 1494 struct nvdimm *nvdimm = to_nvdimm(dev); 1495 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1496 1497 if (nfit_mem->family < 0) 1498 return -ENXIO; 1499 return sprintf(buf, "%#lx\n", nfit_mem->dsm_mask); 1500 } 1501 static DEVICE_ATTR_RO(dsm_mask); 1502 1503 static ssize_t flags_show(struct device *dev, 1504 struct device_attribute *attr, char *buf) 1505 { 1506 u16 flags = to_nfit_memdev(dev)->flags; 1507 1508 return sprintf(buf, "%s%s%s%s%s%s%s\n", 1509 flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "", 1510 flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "", 1511 flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "", 1512 flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "", 1513 flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : "", 1514 flags & ACPI_NFIT_MEM_MAP_FAILED ? "map_fail " : "", 1515 flags & ACPI_NFIT_MEM_HEALTH_ENABLED ? "smart_notify " : ""); 1516 } 1517 static DEVICE_ATTR_RO(flags); 1518 1519 static ssize_t id_show(struct device *dev, 1520 struct device_attribute *attr, char *buf) 1521 { 1522 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1523 1524 if (dcr->valid_fields & ACPI_NFIT_CONTROL_MFG_INFO_VALID) 1525 return sprintf(buf, "%04x-%02x-%04x-%08x\n", 1526 be16_to_cpu(dcr->vendor_id), 1527 dcr->manufacturing_location, 1528 be16_to_cpu(dcr->manufacturing_date), 1529 be32_to_cpu(dcr->serial_number)); 1530 else 1531 return sprintf(buf, "%04x-%08x\n", 1532 be16_to_cpu(dcr->vendor_id), 1533 be32_to_cpu(dcr->serial_number)); 1534 } 1535 static DEVICE_ATTR_RO(id); 1536 1537 static struct attribute *acpi_nfit_dimm_attributes[] = { 1538 &dev_attr_handle.attr, 1539 &dev_attr_phys_id.attr, 1540 &dev_attr_vendor.attr, 1541 &dev_attr_device.attr, 1542 &dev_attr_rev_id.attr, 1543 &dev_attr_subsystem_vendor.attr, 1544 &dev_attr_subsystem_device.attr, 1545 &dev_attr_subsystem_rev_id.attr, 1546 &dev_attr_format.attr, 1547 &dev_attr_formats.attr, 1548 &dev_attr_format1.attr, 1549 &dev_attr_serial.attr, 1550 &dev_attr_flags.attr, 1551 &dev_attr_id.attr, 1552 &dev_attr_family.attr, 1553 &dev_attr_dsm_mask.attr, 1554 NULL, 1555 }; 1556 1557 static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj, 1558 struct attribute *a, int n) 1559 { 1560 struct device *dev = container_of(kobj, struct device, kobj); 1561 struct nvdimm *nvdimm = to_nvdimm(dev); 1562 1563 if (!to_nfit_dcr(dev)) { 1564 /* Without a dcr only the memdev attributes can be surfaced */ 1565 if (a == &dev_attr_handle.attr || a == &dev_attr_phys_id.attr 1566 || a == &dev_attr_flags.attr 1567 || a == &dev_attr_family.attr 1568 || a == &dev_attr_dsm_mask.attr) 1569 return a->mode; 1570 return 0; 1571 } 1572 1573 if (a == &dev_attr_format1.attr && num_nvdimm_formats(nvdimm) <= 1) 1574 return 0; 1575 return a->mode; 1576 } 1577 1578 static const struct attribute_group acpi_nfit_dimm_attribute_group = { 1579 .name = "nfit", 1580 .attrs = acpi_nfit_dimm_attributes, 1581 .is_visible = acpi_nfit_dimm_attr_visible, 1582 }; 1583 1584 static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = { 1585 &nvdimm_attribute_group, 1586 &nd_device_attribute_group, 1587 &acpi_nfit_dimm_attribute_group, 1588 NULL, 1589 }; 1590 1591 static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc, 1592 u32 device_handle) 1593 { 1594 struct nfit_mem *nfit_mem; 1595 1596 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) 1597 if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle) 1598 return nfit_mem->nvdimm; 1599 1600 return NULL; 1601 } 1602 1603 void __acpi_nvdimm_notify(struct device *dev, u32 event) 1604 { 1605 struct nfit_mem *nfit_mem; 1606 struct acpi_nfit_desc *acpi_desc; 1607 1608 dev_dbg(dev->parent, "%s: %s: event: %d\n", dev_name(dev), __func__, 1609 event); 1610 1611 if (event != NFIT_NOTIFY_DIMM_HEALTH) { 1612 dev_dbg(dev->parent, "%s: unknown event: %d\n", dev_name(dev), 1613 event); 1614 return; 1615 } 1616 1617 acpi_desc = dev_get_drvdata(dev->parent); 1618 if (!acpi_desc) 1619 return; 1620 1621 /* 1622 * If we successfully retrieved acpi_desc, then we know nfit_mem data 1623 * is still valid. 1624 */ 1625 nfit_mem = dev_get_drvdata(dev); 1626 if (nfit_mem && nfit_mem->flags_attr) 1627 sysfs_notify_dirent(nfit_mem->flags_attr); 1628 } 1629 EXPORT_SYMBOL_GPL(__acpi_nvdimm_notify); 1630 1631 static void acpi_nvdimm_notify(acpi_handle handle, u32 event, void *data) 1632 { 1633 struct acpi_device *adev = data; 1634 struct device *dev = &adev->dev; 1635 1636 device_lock(dev->parent); 1637 __acpi_nvdimm_notify(dev, event); 1638 device_unlock(dev->parent); 1639 } 1640 1641 static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc, 1642 struct nfit_mem *nfit_mem, u32 device_handle) 1643 { 1644 struct acpi_device *adev, *adev_dimm; 1645 struct device *dev = acpi_desc->dev; 1646 union acpi_object *obj; 1647 unsigned long dsm_mask; 1648 const guid_t *guid; 1649 int i; 1650 int family = -1; 1651 1652 /* nfit test assumes 1:1 relationship between commands and dsms */ 1653 nfit_mem->dsm_mask = acpi_desc->dimm_cmd_force_en; 1654 nfit_mem->family = NVDIMM_FAMILY_INTEL; 1655 adev = to_acpi_dev(acpi_desc); 1656 if (!adev) 1657 return 0; 1658 1659 adev_dimm = acpi_find_child_device(adev, device_handle, false); 1660 nfit_mem->adev = adev_dimm; 1661 if (!adev_dimm) { 1662 dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n", 1663 device_handle); 1664 return force_enable_dimms ? 0 : -ENODEV; 1665 } 1666 1667 if (ACPI_FAILURE(acpi_install_notify_handler(adev_dimm->handle, 1668 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify, adev_dimm))) { 1669 dev_err(dev, "%s: notification registration failed\n", 1670 dev_name(&adev_dimm->dev)); 1671 return -ENXIO; 1672 } 1673 /* 1674 * Record nfit_mem for the notification path to track back to 1675 * the nfit sysfs attributes for this dimm device object. 1676 */ 1677 dev_set_drvdata(&adev_dimm->dev, nfit_mem); 1678 1679 /* 1680 * Until standardization materializes we need to consider 4 1681 * different command sets. Note, that checking for function0 (bit0) 1682 * tells us if any commands are reachable through this GUID. 1683 */ 1684 for (i = 0; i <= NVDIMM_FAMILY_MAX; i++) 1685 if (acpi_check_dsm(adev_dimm->handle, to_nfit_uuid(i), 1, 1)) 1686 if (family < 0 || i == default_dsm_family) 1687 family = i; 1688 1689 /* limit the supported commands to those that are publicly documented */ 1690 nfit_mem->family = family; 1691 if (override_dsm_mask && !disable_vendor_specific) 1692 dsm_mask = override_dsm_mask; 1693 else if (nfit_mem->family == NVDIMM_FAMILY_INTEL) { 1694 dsm_mask = NVDIMM_INTEL_CMDMASK; 1695 if (disable_vendor_specific) 1696 dsm_mask &= ~(1 << ND_CMD_VENDOR); 1697 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE1) { 1698 dsm_mask = 0x1c3c76; 1699 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE2) { 1700 dsm_mask = 0x1fe; 1701 if (disable_vendor_specific) 1702 dsm_mask &= ~(1 << 8); 1703 } else if (nfit_mem->family == NVDIMM_FAMILY_MSFT) { 1704 dsm_mask = 0xffffffff; 1705 } else { 1706 dev_dbg(dev, "unknown dimm command family\n"); 1707 nfit_mem->family = -1; 1708 /* DSMs are optional, continue loading the driver... */ 1709 return 0; 1710 } 1711 1712 guid = to_nfit_uuid(nfit_mem->family); 1713 for_each_set_bit(i, &dsm_mask, BITS_PER_LONG) 1714 if (acpi_check_dsm(adev_dimm->handle, guid, 1715 nfit_dsm_revid(nfit_mem->family, i), 1716 1ULL << i)) 1717 set_bit(i, &nfit_mem->dsm_mask); 1718 1719 obj = acpi_label_info(adev_dimm->handle); 1720 if (obj) { 1721 ACPI_FREE(obj); 1722 nfit_mem->has_lsi = 1; 1723 dev_dbg(dev, "%s: has _LSI\n", dev_name(&adev_dimm->dev)); 1724 } 1725 1726 obj = acpi_label_read(adev_dimm->handle, 0, 0); 1727 if (obj) { 1728 ACPI_FREE(obj); 1729 nfit_mem->has_lsr = 1; 1730 dev_dbg(dev, "%s: has _LSR\n", dev_name(&adev_dimm->dev)); 1731 } 1732 1733 obj = acpi_label_write(adev_dimm->handle, 0, 0, NULL); 1734 if (obj) { 1735 ACPI_FREE(obj); 1736 nfit_mem->has_lsw = 1; 1737 dev_dbg(dev, "%s: has _LSW\n", dev_name(&adev_dimm->dev)); 1738 } 1739 1740 return 0; 1741 } 1742 1743 static void shutdown_dimm_notify(void *data) 1744 { 1745 struct acpi_nfit_desc *acpi_desc = data; 1746 struct nfit_mem *nfit_mem; 1747 1748 mutex_lock(&acpi_desc->init_mutex); 1749 /* 1750 * Clear out the nfit_mem->flags_attr and shut down dimm event 1751 * notifications. 1752 */ 1753 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1754 struct acpi_device *adev_dimm = nfit_mem->adev; 1755 1756 if (nfit_mem->flags_attr) { 1757 sysfs_put(nfit_mem->flags_attr); 1758 nfit_mem->flags_attr = NULL; 1759 } 1760 if (adev_dimm) { 1761 acpi_remove_notify_handler(adev_dimm->handle, 1762 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify); 1763 dev_set_drvdata(&adev_dimm->dev, NULL); 1764 } 1765 } 1766 mutex_unlock(&acpi_desc->init_mutex); 1767 } 1768 1769 static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc) 1770 { 1771 struct nfit_mem *nfit_mem; 1772 int dimm_count = 0, rc; 1773 struct nvdimm *nvdimm; 1774 1775 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1776 struct acpi_nfit_flush_address *flush; 1777 unsigned long flags = 0, cmd_mask; 1778 struct nfit_memdev *nfit_memdev; 1779 u32 device_handle; 1780 u16 mem_flags; 1781 1782 device_handle = __to_nfit_memdev(nfit_mem)->device_handle; 1783 nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle); 1784 if (nvdimm) { 1785 dimm_count++; 1786 continue; 1787 } 1788 1789 if (nfit_mem->bdw && nfit_mem->memdev_pmem) 1790 set_bit(NDD_ALIASING, &flags); 1791 1792 /* collate flags across all memdevs for this dimm */ 1793 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 1794 struct acpi_nfit_memory_map *dimm_memdev; 1795 1796 dimm_memdev = __to_nfit_memdev(nfit_mem); 1797 if (dimm_memdev->device_handle 1798 != nfit_memdev->memdev->device_handle) 1799 continue; 1800 dimm_memdev->flags |= nfit_memdev->memdev->flags; 1801 } 1802 1803 mem_flags = __to_nfit_memdev(nfit_mem)->flags; 1804 if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED) 1805 set_bit(NDD_UNARMED, &flags); 1806 1807 rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle); 1808 if (rc) 1809 continue; 1810 1811 /* 1812 * TODO: provide translation for non-NVDIMM_FAMILY_INTEL 1813 * devices (i.e. from nd_cmd to acpi_dsm) to standardize the 1814 * userspace interface. 1815 */ 1816 cmd_mask = 1UL << ND_CMD_CALL; 1817 if (nfit_mem->family == NVDIMM_FAMILY_INTEL) { 1818 /* 1819 * These commands have a 1:1 correspondence 1820 * between DSM payload and libnvdimm ioctl 1821 * payload format. 1822 */ 1823 cmd_mask |= nfit_mem->dsm_mask & NVDIMM_STANDARD_CMDMASK; 1824 } 1825 1826 if (nfit_mem->has_lsi) 1827 set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask); 1828 if (nfit_mem->has_lsr) 1829 set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask); 1830 if (nfit_mem->has_lsw) 1831 set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask); 1832 1833 flush = nfit_mem->nfit_flush ? nfit_mem->nfit_flush->flush 1834 : NULL; 1835 nvdimm = nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem, 1836 acpi_nfit_dimm_attribute_groups, 1837 flags, cmd_mask, flush ? flush->hint_count : 0, 1838 nfit_mem->flush_wpq); 1839 if (!nvdimm) 1840 return -ENOMEM; 1841 1842 nfit_mem->nvdimm = nvdimm; 1843 dimm_count++; 1844 1845 if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0) 1846 continue; 1847 1848 dev_info(acpi_desc->dev, "%s flags:%s%s%s%s%s\n", 1849 nvdimm_name(nvdimm), 1850 mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "", 1851 mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"", 1852 mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "", 1853 mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : "", 1854 mem_flags & ACPI_NFIT_MEM_MAP_FAILED ? " map_fail" : ""); 1855 1856 } 1857 1858 rc = nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count); 1859 if (rc) 1860 return rc; 1861 1862 /* 1863 * Now that dimms are successfully registered, and async registration 1864 * is flushed, attempt to enable event notification. 1865 */ 1866 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1867 struct kernfs_node *nfit_kernfs; 1868 1869 nvdimm = nfit_mem->nvdimm; 1870 nfit_kernfs = sysfs_get_dirent(nvdimm_kobj(nvdimm)->sd, "nfit"); 1871 if (nfit_kernfs) 1872 nfit_mem->flags_attr = sysfs_get_dirent(nfit_kernfs, 1873 "flags"); 1874 sysfs_put(nfit_kernfs); 1875 if (!nfit_mem->flags_attr) 1876 dev_warn(acpi_desc->dev, "%s: notifications disabled\n", 1877 nvdimm_name(nvdimm)); 1878 } 1879 1880 return devm_add_action_or_reset(acpi_desc->dev, shutdown_dimm_notify, 1881 acpi_desc); 1882 } 1883 1884 /* 1885 * These constants are private because there are no kernel consumers of 1886 * these commands. 1887 */ 1888 enum nfit_aux_cmds { 1889 NFIT_CMD_TRANSLATE_SPA = 5, 1890 NFIT_CMD_ARS_INJECT_SET = 7, 1891 NFIT_CMD_ARS_INJECT_CLEAR = 8, 1892 NFIT_CMD_ARS_INJECT_GET = 9, 1893 }; 1894 1895 static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc) 1896 { 1897 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 1898 const guid_t *guid = to_nfit_uuid(NFIT_DEV_BUS); 1899 struct acpi_device *adev; 1900 unsigned long dsm_mask; 1901 int i; 1902 1903 nd_desc->cmd_mask = acpi_desc->bus_cmd_force_en; 1904 nd_desc->bus_dsm_mask = acpi_desc->bus_nfit_cmd_force_en; 1905 adev = to_acpi_dev(acpi_desc); 1906 if (!adev) 1907 return; 1908 1909 for (i = ND_CMD_ARS_CAP; i <= ND_CMD_CLEAR_ERROR; i++) 1910 if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i)) 1911 set_bit(i, &nd_desc->cmd_mask); 1912 set_bit(ND_CMD_CALL, &nd_desc->cmd_mask); 1913 1914 dsm_mask = 1915 (1 << ND_CMD_ARS_CAP) | 1916 (1 << ND_CMD_ARS_START) | 1917 (1 << ND_CMD_ARS_STATUS) | 1918 (1 << ND_CMD_CLEAR_ERROR) | 1919 (1 << NFIT_CMD_TRANSLATE_SPA) | 1920 (1 << NFIT_CMD_ARS_INJECT_SET) | 1921 (1 << NFIT_CMD_ARS_INJECT_CLEAR) | 1922 (1 << NFIT_CMD_ARS_INJECT_GET); 1923 for_each_set_bit(i, &dsm_mask, BITS_PER_LONG) 1924 if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i)) 1925 set_bit(i, &nd_desc->bus_dsm_mask); 1926 } 1927 1928 static ssize_t range_index_show(struct device *dev, 1929 struct device_attribute *attr, char *buf) 1930 { 1931 struct nd_region *nd_region = to_nd_region(dev); 1932 struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region); 1933 1934 return sprintf(buf, "%d\n", nfit_spa->spa->range_index); 1935 } 1936 static DEVICE_ATTR_RO(range_index); 1937 1938 static ssize_t ecc_unit_size_show(struct device *dev, 1939 struct device_attribute *attr, char *buf) 1940 { 1941 struct nd_region *nd_region = to_nd_region(dev); 1942 struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region); 1943 1944 return sprintf(buf, "%d\n", nfit_spa->clear_err_unit); 1945 } 1946 static DEVICE_ATTR_RO(ecc_unit_size); 1947 1948 static struct attribute *acpi_nfit_region_attributes[] = { 1949 &dev_attr_range_index.attr, 1950 &dev_attr_ecc_unit_size.attr, 1951 NULL, 1952 }; 1953 1954 static const struct attribute_group acpi_nfit_region_attribute_group = { 1955 .name = "nfit", 1956 .attrs = acpi_nfit_region_attributes, 1957 }; 1958 1959 static const struct attribute_group *acpi_nfit_region_attribute_groups[] = { 1960 &nd_region_attribute_group, 1961 &nd_mapping_attribute_group, 1962 &nd_device_attribute_group, 1963 &nd_numa_attribute_group, 1964 &acpi_nfit_region_attribute_group, 1965 NULL, 1966 }; 1967 1968 /* enough info to uniquely specify an interleave set */ 1969 struct nfit_set_info { 1970 struct nfit_set_info_map { 1971 u64 region_offset; 1972 u32 serial_number; 1973 u32 pad; 1974 } mapping[0]; 1975 }; 1976 1977 struct nfit_set_info2 { 1978 struct nfit_set_info_map2 { 1979 u64 region_offset; 1980 u32 serial_number; 1981 u16 vendor_id; 1982 u16 manufacturing_date; 1983 u8 manufacturing_location; 1984 u8 reserved[31]; 1985 } mapping[0]; 1986 }; 1987 1988 static size_t sizeof_nfit_set_info(int num_mappings) 1989 { 1990 return sizeof(struct nfit_set_info) 1991 + num_mappings * sizeof(struct nfit_set_info_map); 1992 } 1993 1994 static size_t sizeof_nfit_set_info2(int num_mappings) 1995 { 1996 return sizeof(struct nfit_set_info2) 1997 + num_mappings * sizeof(struct nfit_set_info_map2); 1998 } 1999 2000 static int cmp_map_compat(const void *m0, const void *m1) 2001 { 2002 const struct nfit_set_info_map *map0 = m0; 2003 const struct nfit_set_info_map *map1 = m1; 2004 2005 return memcmp(&map0->region_offset, &map1->region_offset, 2006 sizeof(u64)); 2007 } 2008 2009 static int cmp_map(const void *m0, const void *m1) 2010 { 2011 const struct nfit_set_info_map *map0 = m0; 2012 const struct nfit_set_info_map *map1 = m1; 2013 2014 if (map0->region_offset < map1->region_offset) 2015 return -1; 2016 else if (map0->region_offset > map1->region_offset) 2017 return 1; 2018 return 0; 2019 } 2020 2021 static int cmp_map2(const void *m0, const void *m1) 2022 { 2023 const struct nfit_set_info_map2 *map0 = m0; 2024 const struct nfit_set_info_map2 *map1 = m1; 2025 2026 if (map0->region_offset < map1->region_offset) 2027 return -1; 2028 else if (map0->region_offset > map1->region_offset) 2029 return 1; 2030 return 0; 2031 } 2032 2033 /* Retrieve the nth entry referencing this spa */ 2034 static struct acpi_nfit_memory_map *memdev_from_spa( 2035 struct acpi_nfit_desc *acpi_desc, u16 range_index, int n) 2036 { 2037 struct nfit_memdev *nfit_memdev; 2038 2039 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) 2040 if (nfit_memdev->memdev->range_index == range_index) 2041 if (n-- == 0) 2042 return nfit_memdev->memdev; 2043 return NULL; 2044 } 2045 2046 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc, 2047 struct nd_region_desc *ndr_desc, 2048 struct acpi_nfit_system_address *spa) 2049 { 2050 struct device *dev = acpi_desc->dev; 2051 struct nd_interleave_set *nd_set; 2052 u16 nr = ndr_desc->num_mappings; 2053 struct nfit_set_info2 *info2; 2054 struct nfit_set_info *info; 2055 int i; 2056 2057 nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL); 2058 if (!nd_set) 2059 return -ENOMEM; 2060 ndr_desc->nd_set = nd_set; 2061 guid_copy(&nd_set->type_guid, (guid_t *) spa->range_guid); 2062 2063 info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL); 2064 if (!info) 2065 return -ENOMEM; 2066 2067 info2 = devm_kzalloc(dev, sizeof_nfit_set_info2(nr), GFP_KERNEL); 2068 if (!info2) 2069 return -ENOMEM; 2070 2071 for (i = 0; i < nr; i++) { 2072 struct nd_mapping_desc *mapping = &ndr_desc->mapping[i]; 2073 struct nfit_set_info_map *map = &info->mapping[i]; 2074 struct nfit_set_info_map2 *map2 = &info2->mapping[i]; 2075 struct nvdimm *nvdimm = mapping->nvdimm; 2076 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 2077 struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc, 2078 spa->range_index, i); 2079 struct acpi_nfit_control_region *dcr = nfit_mem->dcr; 2080 2081 if (!memdev || !nfit_mem->dcr) { 2082 dev_err(dev, "%s: failed to find DCR\n", __func__); 2083 return -ENODEV; 2084 } 2085 2086 map->region_offset = memdev->region_offset; 2087 map->serial_number = dcr->serial_number; 2088 2089 map2->region_offset = memdev->region_offset; 2090 map2->serial_number = dcr->serial_number; 2091 map2->vendor_id = dcr->vendor_id; 2092 map2->manufacturing_date = dcr->manufacturing_date; 2093 map2->manufacturing_location = dcr->manufacturing_location; 2094 } 2095 2096 /* v1.1 namespaces */ 2097 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map), 2098 cmp_map, NULL); 2099 nd_set->cookie1 = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0); 2100 2101 /* v1.2 namespaces */ 2102 sort(&info2->mapping[0], nr, sizeof(struct nfit_set_info_map2), 2103 cmp_map2, NULL); 2104 nd_set->cookie2 = nd_fletcher64(info2, sizeof_nfit_set_info2(nr), 0); 2105 2106 /* support v1.1 namespaces created with the wrong sort order */ 2107 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map), 2108 cmp_map_compat, NULL); 2109 nd_set->altcookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0); 2110 2111 /* record the result of the sort for the mapping position */ 2112 for (i = 0; i < nr; i++) { 2113 struct nfit_set_info_map2 *map2 = &info2->mapping[i]; 2114 int j; 2115 2116 for (j = 0; j < nr; j++) { 2117 struct nd_mapping_desc *mapping = &ndr_desc->mapping[j]; 2118 struct nvdimm *nvdimm = mapping->nvdimm; 2119 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 2120 struct acpi_nfit_control_region *dcr = nfit_mem->dcr; 2121 2122 if (map2->serial_number == dcr->serial_number && 2123 map2->vendor_id == dcr->vendor_id && 2124 map2->manufacturing_date == dcr->manufacturing_date && 2125 map2->manufacturing_location 2126 == dcr->manufacturing_location) { 2127 mapping->position = i; 2128 break; 2129 } 2130 } 2131 } 2132 2133 ndr_desc->nd_set = nd_set; 2134 devm_kfree(dev, info); 2135 devm_kfree(dev, info2); 2136 2137 return 0; 2138 } 2139 2140 static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio) 2141 { 2142 struct acpi_nfit_interleave *idt = mmio->idt; 2143 u32 sub_line_offset, line_index, line_offset; 2144 u64 line_no, table_skip_count, table_offset; 2145 2146 line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset); 2147 table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index); 2148 line_offset = idt->line_offset[line_index] 2149 * mmio->line_size; 2150 table_offset = table_skip_count * mmio->table_size; 2151 2152 return mmio->base_offset + line_offset + table_offset + sub_line_offset; 2153 } 2154 2155 static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw) 2156 { 2157 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; 2158 u64 offset = nfit_blk->stat_offset + mmio->size * bw; 2159 const u32 STATUS_MASK = 0x80000037; 2160 2161 if (mmio->num_lines) 2162 offset = to_interleave_offset(offset, mmio); 2163 2164 return readl(mmio->addr.base + offset) & STATUS_MASK; 2165 } 2166 2167 static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw, 2168 resource_size_t dpa, unsigned int len, unsigned int write) 2169 { 2170 u64 cmd, offset; 2171 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; 2172 2173 enum { 2174 BCW_OFFSET_MASK = (1ULL << 48)-1, 2175 BCW_LEN_SHIFT = 48, 2176 BCW_LEN_MASK = (1ULL << 8) - 1, 2177 BCW_CMD_SHIFT = 56, 2178 }; 2179 2180 cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK; 2181 len = len >> L1_CACHE_SHIFT; 2182 cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT; 2183 cmd |= ((u64) write) << BCW_CMD_SHIFT; 2184 2185 offset = nfit_blk->cmd_offset + mmio->size * bw; 2186 if (mmio->num_lines) 2187 offset = to_interleave_offset(offset, mmio); 2188 2189 writeq(cmd, mmio->addr.base + offset); 2190 nvdimm_flush(nfit_blk->nd_region); 2191 2192 if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH) 2193 readq(mmio->addr.base + offset); 2194 } 2195 2196 static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk, 2197 resource_size_t dpa, void *iobuf, size_t len, int rw, 2198 unsigned int lane) 2199 { 2200 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW]; 2201 unsigned int copied = 0; 2202 u64 base_offset; 2203 int rc; 2204 2205 base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES 2206 + lane * mmio->size; 2207 write_blk_ctl(nfit_blk, lane, dpa, len, rw); 2208 while (len) { 2209 unsigned int c; 2210 u64 offset; 2211 2212 if (mmio->num_lines) { 2213 u32 line_offset; 2214 2215 offset = to_interleave_offset(base_offset + copied, 2216 mmio); 2217 div_u64_rem(offset, mmio->line_size, &line_offset); 2218 c = min_t(size_t, len, mmio->line_size - line_offset); 2219 } else { 2220 offset = base_offset + nfit_blk->bdw_offset; 2221 c = len; 2222 } 2223 2224 if (rw) 2225 memcpy_flushcache(mmio->addr.aperture + offset, iobuf + copied, c); 2226 else { 2227 if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH) 2228 arch_invalidate_pmem((void __force *) 2229 mmio->addr.aperture + offset, c); 2230 2231 memcpy(iobuf + copied, mmio->addr.aperture + offset, c); 2232 } 2233 2234 copied += c; 2235 len -= c; 2236 } 2237 2238 if (rw) 2239 nvdimm_flush(nfit_blk->nd_region); 2240 2241 rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0; 2242 return rc; 2243 } 2244 2245 static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr, 2246 resource_size_t dpa, void *iobuf, u64 len, int rw) 2247 { 2248 struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr); 2249 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW]; 2250 struct nd_region *nd_region = nfit_blk->nd_region; 2251 unsigned int lane, copied = 0; 2252 int rc = 0; 2253 2254 lane = nd_region_acquire_lane(nd_region); 2255 while (len) { 2256 u64 c = min(len, mmio->size); 2257 2258 rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied, 2259 iobuf + copied, c, rw, lane); 2260 if (rc) 2261 break; 2262 2263 copied += c; 2264 len -= c; 2265 } 2266 nd_region_release_lane(nd_region, lane); 2267 2268 return rc; 2269 } 2270 2271 static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio, 2272 struct acpi_nfit_interleave *idt, u16 interleave_ways) 2273 { 2274 if (idt) { 2275 mmio->num_lines = idt->line_count; 2276 mmio->line_size = idt->line_size; 2277 if (interleave_ways == 0) 2278 return -ENXIO; 2279 mmio->table_size = mmio->num_lines * interleave_ways 2280 * mmio->line_size; 2281 } 2282 2283 return 0; 2284 } 2285 2286 static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc, 2287 struct nvdimm *nvdimm, struct nfit_blk *nfit_blk) 2288 { 2289 struct nd_cmd_dimm_flags flags; 2290 int rc; 2291 2292 memset(&flags, 0, sizeof(flags)); 2293 rc = nd_desc->ndctl(nd_desc, nvdimm, ND_CMD_DIMM_FLAGS, &flags, 2294 sizeof(flags), NULL); 2295 2296 if (rc >= 0 && flags.status == 0) 2297 nfit_blk->dimm_flags = flags.flags; 2298 else if (rc == -ENOTTY) { 2299 /* fall back to a conservative default */ 2300 nfit_blk->dimm_flags = NFIT_BLK_DCR_LATCH | NFIT_BLK_READ_FLUSH; 2301 rc = 0; 2302 } else 2303 rc = -ENXIO; 2304 2305 return rc; 2306 } 2307 2308 static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, 2309 struct device *dev) 2310 { 2311 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 2312 struct nd_blk_region *ndbr = to_nd_blk_region(dev); 2313 struct nfit_blk_mmio *mmio; 2314 struct nfit_blk *nfit_blk; 2315 struct nfit_mem *nfit_mem; 2316 struct nvdimm *nvdimm; 2317 int rc; 2318 2319 nvdimm = nd_blk_region_to_dimm(ndbr); 2320 nfit_mem = nvdimm_provider_data(nvdimm); 2321 if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) { 2322 dev_dbg(dev, "%s: missing%s%s%s\n", __func__, 2323 nfit_mem ? "" : " nfit_mem", 2324 (nfit_mem && nfit_mem->dcr) ? "" : " dcr", 2325 (nfit_mem && nfit_mem->bdw) ? "" : " bdw"); 2326 return -ENXIO; 2327 } 2328 2329 nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL); 2330 if (!nfit_blk) 2331 return -ENOMEM; 2332 nd_blk_region_set_provider_data(ndbr, nfit_blk); 2333 nfit_blk->nd_region = to_nd_region(dev); 2334 2335 /* map block aperture memory */ 2336 nfit_blk->bdw_offset = nfit_mem->bdw->offset; 2337 mmio = &nfit_blk->mmio[BDW]; 2338 mmio->addr.base = devm_nvdimm_memremap(dev, nfit_mem->spa_bdw->address, 2339 nfit_mem->spa_bdw->length, nd_blk_memremap_flags(ndbr)); 2340 if (!mmio->addr.base) { 2341 dev_dbg(dev, "%s: %s failed to map bdw\n", __func__, 2342 nvdimm_name(nvdimm)); 2343 return -ENOMEM; 2344 } 2345 mmio->size = nfit_mem->bdw->size; 2346 mmio->base_offset = nfit_mem->memdev_bdw->region_offset; 2347 mmio->idt = nfit_mem->idt_bdw; 2348 mmio->spa = nfit_mem->spa_bdw; 2349 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw, 2350 nfit_mem->memdev_bdw->interleave_ways); 2351 if (rc) { 2352 dev_dbg(dev, "%s: %s failed to init bdw interleave\n", 2353 __func__, nvdimm_name(nvdimm)); 2354 return rc; 2355 } 2356 2357 /* map block control memory */ 2358 nfit_blk->cmd_offset = nfit_mem->dcr->command_offset; 2359 nfit_blk->stat_offset = nfit_mem->dcr->status_offset; 2360 mmio = &nfit_blk->mmio[DCR]; 2361 mmio->addr.base = devm_nvdimm_ioremap(dev, nfit_mem->spa_dcr->address, 2362 nfit_mem->spa_dcr->length); 2363 if (!mmio->addr.base) { 2364 dev_dbg(dev, "%s: %s failed to map dcr\n", __func__, 2365 nvdimm_name(nvdimm)); 2366 return -ENOMEM; 2367 } 2368 mmio->size = nfit_mem->dcr->window_size; 2369 mmio->base_offset = nfit_mem->memdev_dcr->region_offset; 2370 mmio->idt = nfit_mem->idt_dcr; 2371 mmio->spa = nfit_mem->spa_dcr; 2372 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr, 2373 nfit_mem->memdev_dcr->interleave_ways); 2374 if (rc) { 2375 dev_dbg(dev, "%s: %s failed to init dcr interleave\n", 2376 __func__, nvdimm_name(nvdimm)); 2377 return rc; 2378 } 2379 2380 rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk); 2381 if (rc < 0) { 2382 dev_dbg(dev, "%s: %s failed get DIMM flags\n", 2383 __func__, nvdimm_name(nvdimm)); 2384 return rc; 2385 } 2386 2387 if (nvdimm_has_flush(nfit_blk->nd_region) < 0) 2388 dev_warn(dev, "unable to guarantee persistence of writes\n"); 2389 2390 if (mmio->line_size == 0) 2391 return 0; 2392 2393 if ((u32) nfit_blk->cmd_offset % mmio->line_size 2394 + 8 > mmio->line_size) { 2395 dev_dbg(dev, "cmd_offset crosses interleave boundary\n"); 2396 return -ENXIO; 2397 } else if ((u32) nfit_blk->stat_offset % mmio->line_size 2398 + 8 > mmio->line_size) { 2399 dev_dbg(dev, "stat_offset crosses interleave boundary\n"); 2400 return -ENXIO; 2401 } 2402 2403 return 0; 2404 } 2405 2406 static int ars_get_cap(struct acpi_nfit_desc *acpi_desc, 2407 struct nd_cmd_ars_cap *cmd, struct nfit_spa *nfit_spa) 2408 { 2409 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2410 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2411 int cmd_rc, rc; 2412 2413 cmd->address = spa->address; 2414 cmd->length = spa->length; 2415 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, cmd, 2416 sizeof(*cmd), &cmd_rc); 2417 if (rc < 0) 2418 return rc; 2419 return cmd_rc; 2420 } 2421 2422 static int ars_start(struct acpi_nfit_desc *acpi_desc, struct nfit_spa *nfit_spa) 2423 { 2424 int rc; 2425 int cmd_rc; 2426 struct nd_cmd_ars_start ars_start; 2427 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2428 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2429 2430 memset(&ars_start, 0, sizeof(ars_start)); 2431 ars_start.address = spa->address; 2432 ars_start.length = spa->length; 2433 ars_start.flags = acpi_desc->ars_start_flags; 2434 if (nfit_spa_type(spa) == NFIT_SPA_PM) 2435 ars_start.type = ND_ARS_PERSISTENT; 2436 else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE) 2437 ars_start.type = ND_ARS_VOLATILE; 2438 else 2439 return -ENOTTY; 2440 2441 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start, 2442 sizeof(ars_start), &cmd_rc); 2443 2444 if (rc < 0) 2445 return rc; 2446 return cmd_rc; 2447 } 2448 2449 static int ars_continue(struct acpi_nfit_desc *acpi_desc) 2450 { 2451 int rc, cmd_rc; 2452 struct nd_cmd_ars_start ars_start; 2453 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2454 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; 2455 2456 memset(&ars_start, 0, sizeof(ars_start)); 2457 ars_start.address = ars_status->restart_address; 2458 ars_start.length = ars_status->restart_length; 2459 ars_start.type = ars_status->type; 2460 ars_start.flags = acpi_desc->ars_start_flags; 2461 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start, 2462 sizeof(ars_start), &cmd_rc); 2463 if (rc < 0) 2464 return rc; 2465 return cmd_rc; 2466 } 2467 2468 static int ars_get_status(struct acpi_nfit_desc *acpi_desc) 2469 { 2470 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2471 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; 2472 int rc, cmd_rc; 2473 2474 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_STATUS, ars_status, 2475 acpi_desc->ars_status_size, &cmd_rc); 2476 if (rc < 0) 2477 return rc; 2478 return cmd_rc; 2479 } 2480 2481 static int ars_status_process_records(struct acpi_nfit_desc *acpi_desc, 2482 struct nd_cmd_ars_status *ars_status) 2483 { 2484 struct nvdimm_bus *nvdimm_bus = acpi_desc->nvdimm_bus; 2485 int rc; 2486 u32 i; 2487 2488 /* 2489 * First record starts at 44 byte offset from the start of the 2490 * payload. 2491 */ 2492 if (ars_status->out_length < 44) 2493 return 0; 2494 for (i = 0; i < ars_status->num_records; i++) { 2495 /* only process full records */ 2496 if (ars_status->out_length 2497 < 44 + sizeof(struct nd_ars_record) * (i + 1)) 2498 break; 2499 rc = nvdimm_bus_add_badrange(nvdimm_bus, 2500 ars_status->records[i].err_address, 2501 ars_status->records[i].length); 2502 if (rc) 2503 return rc; 2504 } 2505 if (i < ars_status->num_records) 2506 dev_warn(acpi_desc->dev, "detected truncated ars results\n"); 2507 2508 return 0; 2509 } 2510 2511 static void acpi_nfit_remove_resource(void *data) 2512 { 2513 struct resource *res = data; 2514 2515 remove_resource(res); 2516 } 2517 2518 static int acpi_nfit_insert_resource(struct acpi_nfit_desc *acpi_desc, 2519 struct nd_region_desc *ndr_desc) 2520 { 2521 struct resource *res, *nd_res = ndr_desc->res; 2522 int is_pmem, ret; 2523 2524 /* No operation if the region is already registered as PMEM */ 2525 is_pmem = region_intersects(nd_res->start, resource_size(nd_res), 2526 IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY); 2527 if (is_pmem == REGION_INTERSECTS) 2528 return 0; 2529 2530 res = devm_kzalloc(acpi_desc->dev, sizeof(*res), GFP_KERNEL); 2531 if (!res) 2532 return -ENOMEM; 2533 2534 res->name = "Persistent Memory"; 2535 res->start = nd_res->start; 2536 res->end = nd_res->end; 2537 res->flags = IORESOURCE_MEM; 2538 res->desc = IORES_DESC_PERSISTENT_MEMORY; 2539 2540 ret = insert_resource(&iomem_resource, res); 2541 if (ret) 2542 return ret; 2543 2544 ret = devm_add_action_or_reset(acpi_desc->dev, 2545 acpi_nfit_remove_resource, 2546 res); 2547 if (ret) 2548 return ret; 2549 2550 return 0; 2551 } 2552 2553 static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc, 2554 struct nd_mapping_desc *mapping, struct nd_region_desc *ndr_desc, 2555 struct acpi_nfit_memory_map *memdev, 2556 struct nfit_spa *nfit_spa) 2557 { 2558 struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, 2559 memdev->device_handle); 2560 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2561 struct nd_blk_region_desc *ndbr_desc; 2562 struct nfit_mem *nfit_mem; 2563 int blk_valid = 0, rc; 2564 2565 if (!nvdimm) { 2566 dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n", 2567 spa->range_index, memdev->device_handle); 2568 return -ENODEV; 2569 } 2570 2571 mapping->nvdimm = nvdimm; 2572 switch (nfit_spa_type(spa)) { 2573 case NFIT_SPA_PM: 2574 case NFIT_SPA_VOLATILE: 2575 mapping->start = memdev->address; 2576 mapping->size = memdev->region_size; 2577 break; 2578 case NFIT_SPA_DCR: 2579 nfit_mem = nvdimm_provider_data(nvdimm); 2580 if (!nfit_mem || !nfit_mem->bdw) { 2581 dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n", 2582 spa->range_index, nvdimm_name(nvdimm)); 2583 } else { 2584 mapping->size = nfit_mem->bdw->capacity; 2585 mapping->start = nfit_mem->bdw->start_address; 2586 ndr_desc->num_lanes = nfit_mem->bdw->windows; 2587 blk_valid = 1; 2588 } 2589 2590 ndr_desc->mapping = mapping; 2591 ndr_desc->num_mappings = blk_valid; 2592 ndbr_desc = to_blk_region_desc(ndr_desc); 2593 ndbr_desc->enable = acpi_nfit_blk_region_enable; 2594 ndbr_desc->do_io = acpi_desc->blk_do_io; 2595 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa); 2596 if (rc) 2597 return rc; 2598 nfit_spa->nd_region = nvdimm_blk_region_create(acpi_desc->nvdimm_bus, 2599 ndr_desc); 2600 if (!nfit_spa->nd_region) 2601 return -ENOMEM; 2602 break; 2603 } 2604 2605 return 0; 2606 } 2607 2608 static bool nfit_spa_is_virtual(struct acpi_nfit_system_address *spa) 2609 { 2610 return (nfit_spa_type(spa) == NFIT_SPA_VDISK || 2611 nfit_spa_type(spa) == NFIT_SPA_VCD || 2612 nfit_spa_type(spa) == NFIT_SPA_PDISK || 2613 nfit_spa_type(spa) == NFIT_SPA_PCD); 2614 } 2615 2616 static bool nfit_spa_is_volatile(struct acpi_nfit_system_address *spa) 2617 { 2618 return (nfit_spa_type(spa) == NFIT_SPA_VDISK || 2619 nfit_spa_type(spa) == NFIT_SPA_VCD || 2620 nfit_spa_type(spa) == NFIT_SPA_VOLATILE); 2621 } 2622 2623 static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc, 2624 struct nfit_spa *nfit_spa) 2625 { 2626 static struct nd_mapping_desc mappings[ND_MAX_MAPPINGS]; 2627 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2628 struct nd_blk_region_desc ndbr_desc; 2629 struct nd_region_desc *ndr_desc; 2630 struct nfit_memdev *nfit_memdev; 2631 struct nvdimm_bus *nvdimm_bus; 2632 struct resource res; 2633 int count = 0, rc; 2634 2635 if (nfit_spa->nd_region) 2636 return 0; 2637 2638 if (spa->range_index == 0 && !nfit_spa_is_virtual(spa)) { 2639 dev_dbg(acpi_desc->dev, "%s: detected invalid spa index\n", 2640 __func__); 2641 return 0; 2642 } 2643 2644 memset(&res, 0, sizeof(res)); 2645 memset(&mappings, 0, sizeof(mappings)); 2646 memset(&ndbr_desc, 0, sizeof(ndbr_desc)); 2647 res.start = spa->address; 2648 res.end = res.start + spa->length - 1; 2649 ndr_desc = &ndbr_desc.ndr_desc; 2650 ndr_desc->res = &res; 2651 ndr_desc->provider_data = nfit_spa; 2652 ndr_desc->attr_groups = acpi_nfit_region_attribute_groups; 2653 if (spa->flags & ACPI_NFIT_PROXIMITY_VALID) 2654 ndr_desc->numa_node = acpi_map_pxm_to_online_node( 2655 spa->proximity_domain); 2656 else 2657 ndr_desc->numa_node = NUMA_NO_NODE; 2658 2659 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 2660 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev; 2661 struct nd_mapping_desc *mapping; 2662 2663 if (memdev->range_index != spa->range_index) 2664 continue; 2665 if (count >= ND_MAX_MAPPINGS) { 2666 dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n", 2667 spa->range_index, ND_MAX_MAPPINGS); 2668 return -ENXIO; 2669 } 2670 mapping = &mappings[count++]; 2671 rc = acpi_nfit_init_mapping(acpi_desc, mapping, ndr_desc, 2672 memdev, nfit_spa); 2673 if (rc) 2674 goto out; 2675 } 2676 2677 ndr_desc->mapping = mappings; 2678 ndr_desc->num_mappings = count; 2679 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa); 2680 if (rc) 2681 goto out; 2682 2683 nvdimm_bus = acpi_desc->nvdimm_bus; 2684 if (nfit_spa_type(spa) == NFIT_SPA_PM) { 2685 rc = acpi_nfit_insert_resource(acpi_desc, ndr_desc); 2686 if (rc) { 2687 dev_warn(acpi_desc->dev, 2688 "failed to insert pmem resource to iomem: %d\n", 2689 rc); 2690 goto out; 2691 } 2692 2693 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus, 2694 ndr_desc); 2695 if (!nfit_spa->nd_region) 2696 rc = -ENOMEM; 2697 } else if (nfit_spa_is_volatile(spa)) { 2698 nfit_spa->nd_region = nvdimm_volatile_region_create(nvdimm_bus, 2699 ndr_desc); 2700 if (!nfit_spa->nd_region) 2701 rc = -ENOMEM; 2702 } else if (nfit_spa_is_virtual(spa)) { 2703 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus, 2704 ndr_desc); 2705 if (!nfit_spa->nd_region) 2706 rc = -ENOMEM; 2707 } 2708 2709 out: 2710 if (rc) 2711 dev_err(acpi_desc->dev, "failed to register spa range %d\n", 2712 nfit_spa->spa->range_index); 2713 return rc; 2714 } 2715 2716 static int ars_status_alloc(struct acpi_nfit_desc *acpi_desc, 2717 u32 max_ars) 2718 { 2719 struct device *dev = acpi_desc->dev; 2720 struct nd_cmd_ars_status *ars_status; 2721 2722 if (acpi_desc->ars_status && acpi_desc->ars_status_size >= max_ars) { 2723 memset(acpi_desc->ars_status, 0, acpi_desc->ars_status_size); 2724 return 0; 2725 } 2726 2727 if (acpi_desc->ars_status) 2728 devm_kfree(dev, acpi_desc->ars_status); 2729 acpi_desc->ars_status = NULL; 2730 ars_status = devm_kzalloc(dev, max_ars, GFP_KERNEL); 2731 if (!ars_status) 2732 return -ENOMEM; 2733 acpi_desc->ars_status = ars_status; 2734 acpi_desc->ars_status_size = max_ars; 2735 return 0; 2736 } 2737 2738 static int acpi_nfit_query_poison(struct acpi_nfit_desc *acpi_desc, 2739 struct nfit_spa *nfit_spa) 2740 { 2741 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2742 int rc; 2743 2744 if (!nfit_spa->max_ars) { 2745 struct nd_cmd_ars_cap ars_cap; 2746 2747 memset(&ars_cap, 0, sizeof(ars_cap)); 2748 rc = ars_get_cap(acpi_desc, &ars_cap, nfit_spa); 2749 if (rc < 0) 2750 return rc; 2751 nfit_spa->max_ars = ars_cap.max_ars_out; 2752 nfit_spa->clear_err_unit = ars_cap.clear_err_unit; 2753 /* check that the supported scrub types match the spa type */ 2754 if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE && 2755 ((ars_cap.status >> 16) & ND_ARS_VOLATILE) == 0) 2756 return -ENOTTY; 2757 else if (nfit_spa_type(spa) == NFIT_SPA_PM && 2758 ((ars_cap.status >> 16) & ND_ARS_PERSISTENT) == 0) 2759 return -ENOTTY; 2760 } 2761 2762 if (ars_status_alloc(acpi_desc, nfit_spa->max_ars)) 2763 return -ENOMEM; 2764 2765 rc = ars_get_status(acpi_desc); 2766 if (rc < 0 && rc != -ENOSPC) 2767 return rc; 2768 2769 if (ars_status_process_records(acpi_desc, acpi_desc->ars_status)) 2770 return -ENOMEM; 2771 2772 return 0; 2773 } 2774 2775 static void acpi_nfit_async_scrub(struct acpi_nfit_desc *acpi_desc, 2776 struct nfit_spa *nfit_spa) 2777 { 2778 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2779 unsigned int overflow_retry = scrub_overflow_abort; 2780 u64 init_ars_start = 0, init_ars_len = 0; 2781 struct device *dev = acpi_desc->dev; 2782 unsigned int tmo = scrub_timeout; 2783 int rc; 2784 2785 if (!nfit_spa->ars_required || !nfit_spa->nd_region) 2786 return; 2787 2788 rc = ars_start(acpi_desc, nfit_spa); 2789 /* 2790 * If we timed out the initial scan we'll still be busy here, 2791 * and will wait another timeout before giving up permanently. 2792 */ 2793 if (rc < 0 && rc != -EBUSY) 2794 return; 2795 2796 do { 2797 u64 ars_start, ars_len; 2798 2799 if (acpi_desc->cancel) 2800 break; 2801 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa); 2802 if (rc == -ENOTTY) 2803 break; 2804 if (rc == -EBUSY && !tmo) { 2805 dev_warn(dev, "range %d ars timeout, aborting\n", 2806 spa->range_index); 2807 break; 2808 } 2809 2810 if (rc == -EBUSY) { 2811 /* 2812 * Note, entries may be appended to the list 2813 * while the lock is dropped, but the workqueue 2814 * being active prevents entries being deleted / 2815 * freed. 2816 */ 2817 mutex_unlock(&acpi_desc->init_mutex); 2818 ssleep(1); 2819 tmo--; 2820 mutex_lock(&acpi_desc->init_mutex); 2821 continue; 2822 } 2823 2824 /* we got some results, but there are more pending... */ 2825 if (rc == -ENOSPC && overflow_retry--) { 2826 if (!init_ars_len) { 2827 init_ars_len = acpi_desc->ars_status->length; 2828 init_ars_start = acpi_desc->ars_status->address; 2829 } 2830 rc = ars_continue(acpi_desc); 2831 } 2832 2833 if (rc < 0) { 2834 dev_warn(dev, "range %d ars continuation failed\n", 2835 spa->range_index); 2836 break; 2837 } 2838 2839 if (init_ars_len) { 2840 ars_start = init_ars_start; 2841 ars_len = init_ars_len; 2842 } else { 2843 ars_start = acpi_desc->ars_status->address; 2844 ars_len = acpi_desc->ars_status->length; 2845 } 2846 dev_dbg(dev, "spa range: %d ars from %#llx + %#llx complete\n", 2847 spa->range_index, ars_start, ars_len); 2848 /* notify the region about new poison entries */ 2849 nvdimm_region_notify(nfit_spa->nd_region, 2850 NVDIMM_REVALIDATE_POISON); 2851 break; 2852 } while (1); 2853 } 2854 2855 static void acpi_nfit_scrub(struct work_struct *work) 2856 { 2857 struct device *dev; 2858 u64 init_scrub_length = 0; 2859 struct nfit_spa *nfit_spa; 2860 u64 init_scrub_address = 0; 2861 bool init_ars_done = false; 2862 struct acpi_nfit_desc *acpi_desc; 2863 unsigned int tmo = scrub_timeout; 2864 unsigned int overflow_retry = scrub_overflow_abort; 2865 2866 acpi_desc = container_of(work, typeof(*acpi_desc), work); 2867 dev = acpi_desc->dev; 2868 2869 /* 2870 * We scrub in 2 phases. The first phase waits for any platform 2871 * firmware initiated scrubs to complete and then we go search for the 2872 * affected spa regions to mark them scanned. In the second phase we 2873 * initiate a directed scrub for every range that was not scrubbed in 2874 * phase 1. If we're called for a 'rescan', we harmlessly pass through 2875 * the first phase, but really only care about running phase 2, where 2876 * regions can be notified of new poison. 2877 */ 2878 2879 /* process platform firmware initiated scrubs */ 2880 retry: 2881 mutex_lock(&acpi_desc->init_mutex); 2882 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 2883 struct nd_cmd_ars_status *ars_status; 2884 struct acpi_nfit_system_address *spa; 2885 u64 ars_start, ars_len; 2886 int rc; 2887 2888 if (acpi_desc->cancel) 2889 break; 2890 2891 if (nfit_spa->nd_region) 2892 continue; 2893 2894 if (init_ars_done) { 2895 /* 2896 * No need to re-query, we're now just 2897 * reconciling all the ranges covered by the 2898 * initial scrub 2899 */ 2900 rc = 0; 2901 } else 2902 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa); 2903 2904 if (rc == -ENOTTY) { 2905 /* no ars capability, just register spa and move on */ 2906 acpi_nfit_register_region(acpi_desc, nfit_spa); 2907 continue; 2908 } 2909 2910 if (rc == -EBUSY && !tmo) { 2911 /* fallthrough to directed scrub in phase 2 */ 2912 dev_warn(dev, "timeout awaiting ars results, continuing...\n"); 2913 break; 2914 } else if (rc == -EBUSY) { 2915 mutex_unlock(&acpi_desc->init_mutex); 2916 ssleep(1); 2917 tmo--; 2918 goto retry; 2919 } 2920 2921 /* we got some results, but there are more pending... */ 2922 if (rc == -ENOSPC && overflow_retry--) { 2923 ars_status = acpi_desc->ars_status; 2924 /* 2925 * Record the original scrub range, so that we 2926 * can recall all the ranges impacted by the 2927 * initial scrub. 2928 */ 2929 if (!init_scrub_length) { 2930 init_scrub_length = ars_status->length; 2931 init_scrub_address = ars_status->address; 2932 } 2933 rc = ars_continue(acpi_desc); 2934 if (rc == 0) { 2935 mutex_unlock(&acpi_desc->init_mutex); 2936 goto retry; 2937 } 2938 } 2939 2940 if (rc < 0) { 2941 /* 2942 * Initial scrub failed, we'll give it one more 2943 * try below... 2944 */ 2945 break; 2946 } 2947 2948 /* We got some final results, record completed ranges */ 2949 ars_status = acpi_desc->ars_status; 2950 if (init_scrub_length) { 2951 ars_start = init_scrub_address; 2952 ars_len = ars_start + init_scrub_length; 2953 } else { 2954 ars_start = ars_status->address; 2955 ars_len = ars_status->length; 2956 } 2957 spa = nfit_spa->spa; 2958 2959 if (!init_ars_done) { 2960 init_ars_done = true; 2961 dev_dbg(dev, "init scrub %#llx + %#llx complete\n", 2962 ars_start, ars_len); 2963 } 2964 if (ars_start <= spa->address && ars_start + ars_len 2965 >= spa->address + spa->length) 2966 acpi_nfit_register_region(acpi_desc, nfit_spa); 2967 } 2968 2969 /* 2970 * For all the ranges not covered by an initial scrub we still 2971 * want to see if there are errors, but it's ok to discover them 2972 * asynchronously. 2973 */ 2974 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 2975 /* 2976 * Flag all the ranges that still need scrubbing, but 2977 * register them now to make data available. 2978 */ 2979 if (!nfit_spa->nd_region) { 2980 nfit_spa->ars_required = 1; 2981 acpi_nfit_register_region(acpi_desc, nfit_spa); 2982 } 2983 } 2984 acpi_desc->init_complete = 1; 2985 2986 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) 2987 acpi_nfit_async_scrub(acpi_desc, nfit_spa); 2988 acpi_desc->scrub_count++; 2989 acpi_desc->ars_start_flags = 0; 2990 if (acpi_desc->scrub_count_state) 2991 sysfs_notify_dirent(acpi_desc->scrub_count_state); 2992 mutex_unlock(&acpi_desc->init_mutex); 2993 } 2994 2995 static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc) 2996 { 2997 struct nfit_spa *nfit_spa; 2998 int rc; 2999 3000 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) 3001 if (nfit_spa_type(nfit_spa->spa) == NFIT_SPA_DCR) { 3002 /* BLK regions don't need to wait for ars results */ 3003 rc = acpi_nfit_register_region(acpi_desc, nfit_spa); 3004 if (rc) 3005 return rc; 3006 } 3007 3008 acpi_desc->ars_start_flags = 0; 3009 if (!acpi_desc->cancel) 3010 queue_work(nfit_wq, &acpi_desc->work); 3011 return 0; 3012 } 3013 3014 static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc, 3015 struct nfit_table_prev *prev) 3016 { 3017 struct device *dev = acpi_desc->dev; 3018 3019 if (!list_empty(&prev->spas) || 3020 !list_empty(&prev->memdevs) || 3021 !list_empty(&prev->dcrs) || 3022 !list_empty(&prev->bdws) || 3023 !list_empty(&prev->idts) || 3024 !list_empty(&prev->flushes)) { 3025 dev_err(dev, "new nfit deletes entries (unsupported)\n"); 3026 return -ENXIO; 3027 } 3028 return 0; 3029 } 3030 3031 static int acpi_nfit_desc_init_scrub_attr(struct acpi_nfit_desc *acpi_desc) 3032 { 3033 struct device *dev = acpi_desc->dev; 3034 struct kernfs_node *nfit; 3035 struct device *bus_dev; 3036 3037 if (!ars_supported(acpi_desc->nvdimm_bus)) 3038 return 0; 3039 3040 bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); 3041 nfit = sysfs_get_dirent(bus_dev->kobj.sd, "nfit"); 3042 if (!nfit) { 3043 dev_err(dev, "sysfs_get_dirent 'nfit' failed\n"); 3044 return -ENODEV; 3045 } 3046 acpi_desc->scrub_count_state = sysfs_get_dirent(nfit, "scrub"); 3047 sysfs_put(nfit); 3048 if (!acpi_desc->scrub_count_state) { 3049 dev_err(dev, "sysfs_get_dirent 'scrub' failed\n"); 3050 return -ENODEV; 3051 } 3052 3053 return 0; 3054 } 3055 3056 static void acpi_nfit_unregister(void *data) 3057 { 3058 struct acpi_nfit_desc *acpi_desc = data; 3059 3060 nvdimm_bus_unregister(acpi_desc->nvdimm_bus); 3061 } 3062 3063 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz) 3064 { 3065 struct device *dev = acpi_desc->dev; 3066 struct nfit_table_prev prev; 3067 const void *end; 3068 int rc; 3069 3070 if (!acpi_desc->nvdimm_bus) { 3071 acpi_nfit_init_dsms(acpi_desc); 3072 3073 acpi_desc->nvdimm_bus = nvdimm_bus_register(dev, 3074 &acpi_desc->nd_desc); 3075 if (!acpi_desc->nvdimm_bus) 3076 return -ENOMEM; 3077 3078 rc = devm_add_action_or_reset(dev, acpi_nfit_unregister, 3079 acpi_desc); 3080 if (rc) 3081 return rc; 3082 3083 rc = acpi_nfit_desc_init_scrub_attr(acpi_desc); 3084 if (rc) 3085 return rc; 3086 3087 /* register this acpi_desc for mce notifications */ 3088 mutex_lock(&acpi_desc_lock); 3089 list_add_tail(&acpi_desc->list, &acpi_descs); 3090 mutex_unlock(&acpi_desc_lock); 3091 } 3092 3093 mutex_lock(&acpi_desc->init_mutex); 3094 3095 INIT_LIST_HEAD(&prev.spas); 3096 INIT_LIST_HEAD(&prev.memdevs); 3097 INIT_LIST_HEAD(&prev.dcrs); 3098 INIT_LIST_HEAD(&prev.bdws); 3099 INIT_LIST_HEAD(&prev.idts); 3100 INIT_LIST_HEAD(&prev.flushes); 3101 3102 list_cut_position(&prev.spas, &acpi_desc->spas, 3103 acpi_desc->spas.prev); 3104 list_cut_position(&prev.memdevs, &acpi_desc->memdevs, 3105 acpi_desc->memdevs.prev); 3106 list_cut_position(&prev.dcrs, &acpi_desc->dcrs, 3107 acpi_desc->dcrs.prev); 3108 list_cut_position(&prev.bdws, &acpi_desc->bdws, 3109 acpi_desc->bdws.prev); 3110 list_cut_position(&prev.idts, &acpi_desc->idts, 3111 acpi_desc->idts.prev); 3112 list_cut_position(&prev.flushes, &acpi_desc->flushes, 3113 acpi_desc->flushes.prev); 3114 3115 end = data + sz; 3116 while (!IS_ERR_OR_NULL(data)) 3117 data = add_table(acpi_desc, &prev, data, end); 3118 3119 if (IS_ERR(data)) { 3120 dev_dbg(dev, "%s: nfit table parsing error: %ld\n", __func__, 3121 PTR_ERR(data)); 3122 rc = PTR_ERR(data); 3123 goto out_unlock; 3124 } 3125 3126 rc = acpi_nfit_check_deletions(acpi_desc, &prev); 3127 if (rc) 3128 goto out_unlock; 3129 3130 rc = nfit_mem_init(acpi_desc); 3131 if (rc) 3132 goto out_unlock; 3133 3134 rc = acpi_nfit_register_dimms(acpi_desc); 3135 if (rc) 3136 goto out_unlock; 3137 3138 rc = acpi_nfit_register_regions(acpi_desc); 3139 3140 out_unlock: 3141 mutex_unlock(&acpi_desc->init_mutex); 3142 return rc; 3143 } 3144 EXPORT_SYMBOL_GPL(acpi_nfit_init); 3145 3146 struct acpi_nfit_flush_work { 3147 struct work_struct work; 3148 struct completion cmp; 3149 }; 3150 3151 static void flush_probe(struct work_struct *work) 3152 { 3153 struct acpi_nfit_flush_work *flush; 3154 3155 flush = container_of(work, typeof(*flush), work); 3156 complete(&flush->cmp); 3157 } 3158 3159 static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc) 3160 { 3161 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 3162 struct device *dev = acpi_desc->dev; 3163 struct acpi_nfit_flush_work flush; 3164 int rc; 3165 3166 /* bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */ 3167 device_lock(dev); 3168 device_unlock(dev); 3169 3170 /* bounce the init_mutex to make init_complete valid */ 3171 mutex_lock(&acpi_desc->init_mutex); 3172 if (acpi_desc->cancel || acpi_desc->init_complete) { 3173 mutex_unlock(&acpi_desc->init_mutex); 3174 return 0; 3175 } 3176 3177 /* 3178 * Scrub work could take 10s of seconds, userspace may give up so we 3179 * need to be interruptible while waiting. 3180 */ 3181 INIT_WORK_ONSTACK(&flush.work, flush_probe); 3182 init_completion(&flush.cmp); 3183 queue_work(nfit_wq, &flush.work); 3184 mutex_unlock(&acpi_desc->init_mutex); 3185 3186 rc = wait_for_completion_interruptible(&flush.cmp); 3187 cancel_work_sync(&flush.work); 3188 return rc; 3189 } 3190 3191 static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc, 3192 struct nvdimm *nvdimm, unsigned int cmd) 3193 { 3194 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 3195 3196 if (nvdimm) 3197 return 0; 3198 if (cmd != ND_CMD_ARS_START) 3199 return 0; 3200 3201 /* 3202 * The kernel and userspace may race to initiate a scrub, but 3203 * the scrub thread is prepared to lose that initial race. It 3204 * just needs guarantees that any ars it initiates are not 3205 * interrupted by any intervening start reqeusts from userspace. 3206 */ 3207 if (work_busy(&acpi_desc->work)) 3208 return -EBUSY; 3209 3210 return 0; 3211 } 3212 3213 int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc, u8 flags) 3214 { 3215 struct device *dev = acpi_desc->dev; 3216 struct nfit_spa *nfit_spa; 3217 3218 if (work_busy(&acpi_desc->work)) 3219 return -EBUSY; 3220 3221 mutex_lock(&acpi_desc->init_mutex); 3222 if (acpi_desc->cancel) { 3223 mutex_unlock(&acpi_desc->init_mutex); 3224 return 0; 3225 } 3226 3227 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 3228 struct acpi_nfit_system_address *spa = nfit_spa->spa; 3229 3230 if (nfit_spa_type(spa) != NFIT_SPA_PM) 3231 continue; 3232 3233 nfit_spa->ars_required = 1; 3234 } 3235 acpi_desc->ars_start_flags = flags; 3236 queue_work(nfit_wq, &acpi_desc->work); 3237 dev_dbg(dev, "%s: ars_scan triggered\n", __func__); 3238 mutex_unlock(&acpi_desc->init_mutex); 3239 3240 return 0; 3241 } 3242 3243 void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev) 3244 { 3245 struct nvdimm_bus_descriptor *nd_desc; 3246 3247 dev_set_drvdata(dev, acpi_desc); 3248 acpi_desc->dev = dev; 3249 acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io; 3250 nd_desc = &acpi_desc->nd_desc; 3251 nd_desc->provider_name = "ACPI.NFIT"; 3252 nd_desc->module = THIS_MODULE; 3253 nd_desc->ndctl = acpi_nfit_ctl; 3254 nd_desc->flush_probe = acpi_nfit_flush_probe; 3255 nd_desc->clear_to_send = acpi_nfit_clear_to_send; 3256 nd_desc->attr_groups = acpi_nfit_attribute_groups; 3257 3258 INIT_LIST_HEAD(&acpi_desc->spas); 3259 INIT_LIST_HEAD(&acpi_desc->dcrs); 3260 INIT_LIST_HEAD(&acpi_desc->bdws); 3261 INIT_LIST_HEAD(&acpi_desc->idts); 3262 INIT_LIST_HEAD(&acpi_desc->flushes); 3263 INIT_LIST_HEAD(&acpi_desc->memdevs); 3264 INIT_LIST_HEAD(&acpi_desc->dimms); 3265 INIT_LIST_HEAD(&acpi_desc->list); 3266 mutex_init(&acpi_desc->init_mutex); 3267 INIT_WORK(&acpi_desc->work, acpi_nfit_scrub); 3268 } 3269 EXPORT_SYMBOL_GPL(acpi_nfit_desc_init); 3270 3271 static void acpi_nfit_put_table(void *table) 3272 { 3273 acpi_put_table(table); 3274 } 3275 3276 void acpi_nfit_shutdown(void *data) 3277 { 3278 struct acpi_nfit_desc *acpi_desc = data; 3279 struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); 3280 3281 /* 3282 * Destruct under acpi_desc_lock so that nfit_handle_mce does not 3283 * race teardown 3284 */ 3285 mutex_lock(&acpi_desc_lock); 3286 list_del(&acpi_desc->list); 3287 mutex_unlock(&acpi_desc_lock); 3288 3289 mutex_lock(&acpi_desc->init_mutex); 3290 acpi_desc->cancel = 1; 3291 mutex_unlock(&acpi_desc->init_mutex); 3292 3293 /* 3294 * Bounce the nvdimm bus lock to make sure any in-flight 3295 * acpi_nfit_ars_rescan() submissions have had a chance to 3296 * either submit or see ->cancel set. 3297 */ 3298 device_lock(bus_dev); 3299 device_unlock(bus_dev); 3300 3301 flush_workqueue(nfit_wq); 3302 } 3303 EXPORT_SYMBOL_GPL(acpi_nfit_shutdown); 3304 3305 static int acpi_nfit_add(struct acpi_device *adev) 3306 { 3307 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 3308 struct acpi_nfit_desc *acpi_desc; 3309 struct device *dev = &adev->dev; 3310 struct acpi_table_header *tbl; 3311 acpi_status status = AE_OK; 3312 acpi_size sz; 3313 int rc = 0; 3314 3315 status = acpi_get_table(ACPI_SIG_NFIT, 0, &tbl); 3316 if (ACPI_FAILURE(status)) { 3317 /* This is ok, we could have an nvdimm hotplugged later */ 3318 dev_dbg(dev, "failed to find NFIT at startup\n"); 3319 return 0; 3320 } 3321 3322 rc = devm_add_action_or_reset(dev, acpi_nfit_put_table, tbl); 3323 if (rc) 3324 return rc; 3325 sz = tbl->length; 3326 3327 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL); 3328 if (!acpi_desc) 3329 return -ENOMEM; 3330 acpi_nfit_desc_init(acpi_desc, &adev->dev); 3331 3332 /* Save the acpi header for exporting the revision via sysfs */ 3333 acpi_desc->acpi_header = *tbl; 3334 3335 /* Evaluate _FIT and override with that if present */ 3336 status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf); 3337 if (ACPI_SUCCESS(status) && buf.length > 0) { 3338 union acpi_object *obj = buf.pointer; 3339 3340 if (obj->type == ACPI_TYPE_BUFFER) 3341 rc = acpi_nfit_init(acpi_desc, obj->buffer.pointer, 3342 obj->buffer.length); 3343 else 3344 dev_dbg(dev, "%s invalid type %d, ignoring _FIT\n", 3345 __func__, (int) obj->type); 3346 kfree(buf.pointer); 3347 } else 3348 /* skip over the lead-in header table */ 3349 rc = acpi_nfit_init(acpi_desc, (void *) tbl 3350 + sizeof(struct acpi_table_nfit), 3351 sz - sizeof(struct acpi_table_nfit)); 3352 3353 if (rc) 3354 return rc; 3355 return devm_add_action_or_reset(dev, acpi_nfit_shutdown, acpi_desc); 3356 } 3357 3358 static int acpi_nfit_remove(struct acpi_device *adev) 3359 { 3360 /* see acpi_nfit_unregister */ 3361 return 0; 3362 } 3363 3364 static void acpi_nfit_update_notify(struct device *dev, acpi_handle handle) 3365 { 3366 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev); 3367 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 3368 union acpi_object *obj; 3369 acpi_status status; 3370 int ret; 3371 3372 if (!dev->driver) { 3373 /* dev->driver may be null if we're being removed */ 3374 dev_dbg(dev, "%s: no driver found for dev\n", __func__); 3375 return; 3376 } 3377 3378 if (!acpi_desc) { 3379 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL); 3380 if (!acpi_desc) 3381 return; 3382 acpi_nfit_desc_init(acpi_desc, dev); 3383 } else { 3384 /* 3385 * Finish previous registration before considering new 3386 * regions. 3387 */ 3388 flush_workqueue(nfit_wq); 3389 } 3390 3391 /* Evaluate _FIT */ 3392 status = acpi_evaluate_object(handle, "_FIT", NULL, &buf); 3393 if (ACPI_FAILURE(status)) { 3394 dev_err(dev, "failed to evaluate _FIT\n"); 3395 return; 3396 } 3397 3398 obj = buf.pointer; 3399 if (obj->type == ACPI_TYPE_BUFFER) { 3400 ret = acpi_nfit_init(acpi_desc, obj->buffer.pointer, 3401 obj->buffer.length); 3402 if (ret) 3403 dev_err(dev, "failed to merge updated NFIT\n"); 3404 } else 3405 dev_err(dev, "Invalid _FIT\n"); 3406 kfree(buf.pointer); 3407 } 3408 3409 static void acpi_nfit_uc_error_notify(struct device *dev, acpi_handle handle) 3410 { 3411 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev); 3412 u8 flags = (acpi_desc->scrub_mode == HW_ERROR_SCRUB_ON) ? 3413 0 : ND_ARS_RETURN_PREV_DATA; 3414 3415 acpi_nfit_ars_rescan(acpi_desc, flags); 3416 } 3417 3418 void __acpi_nfit_notify(struct device *dev, acpi_handle handle, u32 event) 3419 { 3420 dev_dbg(dev, "%s: event: 0x%x\n", __func__, event); 3421 3422 switch (event) { 3423 case NFIT_NOTIFY_UPDATE: 3424 return acpi_nfit_update_notify(dev, handle); 3425 case NFIT_NOTIFY_UC_MEMORY_ERROR: 3426 return acpi_nfit_uc_error_notify(dev, handle); 3427 default: 3428 return; 3429 } 3430 } 3431 EXPORT_SYMBOL_GPL(__acpi_nfit_notify); 3432 3433 static void acpi_nfit_notify(struct acpi_device *adev, u32 event) 3434 { 3435 device_lock(&adev->dev); 3436 __acpi_nfit_notify(&adev->dev, adev->handle, event); 3437 device_unlock(&adev->dev); 3438 } 3439 3440 static const struct acpi_device_id acpi_nfit_ids[] = { 3441 { "ACPI0012", 0 }, 3442 { "", 0 }, 3443 }; 3444 MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids); 3445 3446 static struct acpi_driver acpi_nfit_driver = { 3447 .name = KBUILD_MODNAME, 3448 .ids = acpi_nfit_ids, 3449 .ops = { 3450 .add = acpi_nfit_add, 3451 .remove = acpi_nfit_remove, 3452 .notify = acpi_nfit_notify, 3453 }, 3454 }; 3455 3456 static __init int nfit_init(void) 3457 { 3458 int ret; 3459 3460 BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40); 3461 BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56); 3462 BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48); 3463 BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20); 3464 BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9); 3465 BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80); 3466 BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40); 3467 3468 guid_parse(UUID_VOLATILE_MEMORY, &nfit_uuid[NFIT_SPA_VOLATILE]); 3469 guid_parse(UUID_PERSISTENT_MEMORY, &nfit_uuid[NFIT_SPA_PM]); 3470 guid_parse(UUID_CONTROL_REGION, &nfit_uuid[NFIT_SPA_DCR]); 3471 guid_parse(UUID_DATA_REGION, &nfit_uuid[NFIT_SPA_BDW]); 3472 guid_parse(UUID_VOLATILE_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_VDISK]); 3473 guid_parse(UUID_VOLATILE_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_VCD]); 3474 guid_parse(UUID_PERSISTENT_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_PDISK]); 3475 guid_parse(UUID_PERSISTENT_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_PCD]); 3476 guid_parse(UUID_NFIT_BUS, &nfit_uuid[NFIT_DEV_BUS]); 3477 guid_parse(UUID_NFIT_DIMM, &nfit_uuid[NFIT_DEV_DIMM]); 3478 guid_parse(UUID_NFIT_DIMM_N_HPE1, &nfit_uuid[NFIT_DEV_DIMM_N_HPE1]); 3479 guid_parse(UUID_NFIT_DIMM_N_HPE2, &nfit_uuid[NFIT_DEV_DIMM_N_HPE2]); 3480 guid_parse(UUID_NFIT_DIMM_N_MSFT, &nfit_uuid[NFIT_DEV_DIMM_N_MSFT]); 3481 3482 nfit_wq = create_singlethread_workqueue("nfit"); 3483 if (!nfit_wq) 3484 return -ENOMEM; 3485 3486 nfit_mce_register(); 3487 ret = acpi_bus_register_driver(&acpi_nfit_driver); 3488 if (ret) { 3489 nfit_mce_unregister(); 3490 destroy_workqueue(nfit_wq); 3491 } 3492 3493 return ret; 3494 3495 } 3496 3497 static __exit void nfit_exit(void) 3498 { 3499 nfit_mce_unregister(); 3500 acpi_bus_unregister_driver(&acpi_nfit_driver); 3501 destroy_workqueue(nfit_wq); 3502 WARN_ON(!list_empty(&acpi_descs)); 3503 } 3504 3505 module_init(nfit_init); 3506 module_exit(nfit_exit); 3507 MODULE_LICENSE("GPL v2"); 3508 MODULE_AUTHOR("Intel Corporation"); 3509