1 /* 2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of version 2 of the GNU General Public License as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 */ 13 #include <linux/list_sort.h> 14 #include <linux/libnvdimm.h> 15 #include <linux/module.h> 16 #include <linux/mutex.h> 17 #include <linux/ndctl.h> 18 #include <linux/sysfs.h> 19 #include <linux/delay.h> 20 #include <linux/list.h> 21 #include <linux/acpi.h> 22 #include <linux/sort.h> 23 #include <linux/io.h> 24 #include <linux/nd.h> 25 #include <asm/cacheflush.h> 26 #include "nfit.h" 27 28 /* 29 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is 30 * irrelevant. 31 */ 32 #include <linux/io-64-nonatomic-hi-lo.h> 33 34 static bool force_enable_dimms; 35 module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR); 36 MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status"); 37 38 static unsigned int scrub_timeout = NFIT_ARS_TIMEOUT; 39 module_param(scrub_timeout, uint, S_IRUGO|S_IWUSR); 40 MODULE_PARM_DESC(scrub_timeout, "Initial scrub timeout in seconds"); 41 42 /* after three payloads of overflow, it's dead jim */ 43 static unsigned int scrub_overflow_abort = 3; 44 module_param(scrub_overflow_abort, uint, S_IRUGO|S_IWUSR); 45 MODULE_PARM_DESC(scrub_overflow_abort, 46 "Number of times we overflow ARS results before abort"); 47 48 static bool disable_vendor_specific; 49 module_param(disable_vendor_specific, bool, S_IRUGO); 50 MODULE_PARM_DESC(disable_vendor_specific, 51 "Limit commands to the publicly specified set"); 52 53 static unsigned long override_dsm_mask; 54 module_param(override_dsm_mask, ulong, S_IRUGO); 55 MODULE_PARM_DESC(override_dsm_mask, "Bitmask of allowed NVDIMM DSM functions"); 56 57 static int default_dsm_family = -1; 58 module_param(default_dsm_family, int, S_IRUGO); 59 MODULE_PARM_DESC(default_dsm_family, 60 "Try this DSM type first when identifying NVDIMM family"); 61 62 LIST_HEAD(acpi_descs); 63 DEFINE_MUTEX(acpi_desc_lock); 64 65 static struct workqueue_struct *nfit_wq; 66 67 struct nfit_table_prev { 68 struct list_head spas; 69 struct list_head memdevs; 70 struct list_head dcrs; 71 struct list_head bdws; 72 struct list_head idts; 73 struct list_head flushes; 74 }; 75 76 static guid_t nfit_uuid[NFIT_UUID_MAX]; 77 78 const guid_t *to_nfit_uuid(enum nfit_uuids id) 79 { 80 return &nfit_uuid[id]; 81 } 82 EXPORT_SYMBOL(to_nfit_uuid); 83 84 static struct acpi_nfit_desc *to_acpi_nfit_desc( 85 struct nvdimm_bus_descriptor *nd_desc) 86 { 87 return container_of(nd_desc, struct acpi_nfit_desc, nd_desc); 88 } 89 90 static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc) 91 { 92 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 93 94 /* 95 * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct 96 * acpi_device. 97 */ 98 if (!nd_desc->provider_name 99 || strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0) 100 return NULL; 101 102 return to_acpi_device(acpi_desc->dev); 103 } 104 105 static int xlat_bus_status(void *buf, unsigned int cmd, u32 status) 106 { 107 struct nd_cmd_clear_error *clear_err; 108 struct nd_cmd_ars_status *ars_status; 109 u16 flags; 110 111 switch (cmd) { 112 case ND_CMD_ARS_CAP: 113 if ((status & 0xffff) == NFIT_ARS_CAP_NONE) 114 return -ENOTTY; 115 116 /* Command failed */ 117 if (status & 0xffff) 118 return -EIO; 119 120 /* No supported scan types for this range */ 121 flags = ND_ARS_PERSISTENT | ND_ARS_VOLATILE; 122 if ((status >> 16 & flags) == 0) 123 return -ENOTTY; 124 return 0; 125 case ND_CMD_ARS_START: 126 /* ARS is in progress */ 127 if ((status & 0xffff) == NFIT_ARS_START_BUSY) 128 return -EBUSY; 129 130 /* Command failed */ 131 if (status & 0xffff) 132 return -EIO; 133 return 0; 134 case ND_CMD_ARS_STATUS: 135 ars_status = buf; 136 /* Command failed */ 137 if (status & 0xffff) 138 return -EIO; 139 /* Check extended status (Upper two bytes) */ 140 if (status == NFIT_ARS_STATUS_DONE) 141 return 0; 142 143 /* ARS is in progress */ 144 if (status == NFIT_ARS_STATUS_BUSY) 145 return -EBUSY; 146 147 /* No ARS performed for the current boot */ 148 if (status == NFIT_ARS_STATUS_NONE) 149 return -EAGAIN; 150 151 /* 152 * ARS interrupted, either we overflowed or some other 153 * agent wants the scan to stop. If we didn't overflow 154 * then just continue with the returned results. 155 */ 156 if (status == NFIT_ARS_STATUS_INTR) { 157 if (ars_status->out_length >= 40 && (ars_status->flags 158 & NFIT_ARS_F_OVERFLOW)) 159 return -ENOSPC; 160 return 0; 161 } 162 163 /* Unknown status */ 164 if (status >> 16) 165 return -EIO; 166 return 0; 167 case ND_CMD_CLEAR_ERROR: 168 clear_err = buf; 169 if (status & 0xffff) 170 return -EIO; 171 if (!clear_err->cleared) 172 return -EIO; 173 if (clear_err->length > clear_err->cleared) 174 return clear_err->cleared; 175 return 0; 176 default: 177 break; 178 } 179 180 /* all other non-zero status results in an error */ 181 if (status) 182 return -EIO; 183 return 0; 184 } 185 186 #define ACPI_LABELS_LOCKED 3 187 188 static int xlat_nvdimm_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd, 189 u32 status) 190 { 191 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 192 193 switch (cmd) { 194 case ND_CMD_GET_CONFIG_SIZE: 195 /* 196 * In the _LSI, _LSR, _LSW case the locked status is 197 * communicated via the read/write commands 198 */ 199 if (nfit_mem->has_lsi) 200 break; 201 202 if (status >> 16 & ND_CONFIG_LOCKED) 203 return -EACCES; 204 break; 205 case ND_CMD_GET_CONFIG_DATA: 206 if (nfit_mem->has_lsr && status == ACPI_LABELS_LOCKED) 207 return -EACCES; 208 break; 209 case ND_CMD_SET_CONFIG_DATA: 210 if (nfit_mem->has_lsw && status == ACPI_LABELS_LOCKED) 211 return -EACCES; 212 break; 213 default: 214 break; 215 } 216 217 /* all other non-zero status results in an error */ 218 if (status) 219 return -EIO; 220 return 0; 221 } 222 223 static int xlat_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd, 224 u32 status) 225 { 226 if (!nvdimm) 227 return xlat_bus_status(buf, cmd, status); 228 return xlat_nvdimm_status(nvdimm, buf, cmd, status); 229 } 230 231 /* convert _LS{I,R} packages to the buffer object acpi_nfit_ctl expects */ 232 static union acpi_object *pkg_to_buf(union acpi_object *pkg) 233 { 234 int i; 235 void *dst; 236 size_t size = 0; 237 union acpi_object *buf = NULL; 238 239 if (pkg->type != ACPI_TYPE_PACKAGE) { 240 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n", 241 pkg->type); 242 goto err; 243 } 244 245 for (i = 0; i < pkg->package.count; i++) { 246 union acpi_object *obj = &pkg->package.elements[i]; 247 248 if (obj->type == ACPI_TYPE_INTEGER) 249 size += 4; 250 else if (obj->type == ACPI_TYPE_BUFFER) 251 size += obj->buffer.length; 252 else { 253 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n", 254 obj->type); 255 goto err; 256 } 257 } 258 259 buf = ACPI_ALLOCATE(sizeof(*buf) + size); 260 if (!buf) 261 goto err; 262 263 dst = buf + 1; 264 buf->type = ACPI_TYPE_BUFFER; 265 buf->buffer.length = size; 266 buf->buffer.pointer = dst; 267 for (i = 0; i < pkg->package.count; i++) { 268 union acpi_object *obj = &pkg->package.elements[i]; 269 270 if (obj->type == ACPI_TYPE_INTEGER) { 271 memcpy(dst, &obj->integer.value, 4); 272 dst += 4; 273 } else if (obj->type == ACPI_TYPE_BUFFER) { 274 memcpy(dst, obj->buffer.pointer, obj->buffer.length); 275 dst += obj->buffer.length; 276 } 277 } 278 err: 279 ACPI_FREE(pkg); 280 return buf; 281 } 282 283 static union acpi_object *int_to_buf(union acpi_object *integer) 284 { 285 union acpi_object *buf = ACPI_ALLOCATE(sizeof(*buf) + 4); 286 void *dst = NULL; 287 288 if (!buf) 289 goto err; 290 291 if (integer->type != ACPI_TYPE_INTEGER) { 292 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n", 293 integer->type); 294 goto err; 295 } 296 297 dst = buf + 1; 298 buf->type = ACPI_TYPE_BUFFER; 299 buf->buffer.length = 4; 300 buf->buffer.pointer = dst; 301 memcpy(dst, &integer->integer.value, 4); 302 err: 303 ACPI_FREE(integer); 304 return buf; 305 } 306 307 static union acpi_object *acpi_label_write(acpi_handle handle, u32 offset, 308 u32 len, void *data) 309 { 310 acpi_status rc; 311 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 312 struct acpi_object_list input = { 313 .count = 3, 314 .pointer = (union acpi_object []) { 315 [0] = { 316 .integer.type = ACPI_TYPE_INTEGER, 317 .integer.value = offset, 318 }, 319 [1] = { 320 .integer.type = ACPI_TYPE_INTEGER, 321 .integer.value = len, 322 }, 323 [2] = { 324 .buffer.type = ACPI_TYPE_BUFFER, 325 .buffer.pointer = data, 326 .buffer.length = len, 327 }, 328 }, 329 }; 330 331 rc = acpi_evaluate_object(handle, "_LSW", &input, &buf); 332 if (ACPI_FAILURE(rc)) 333 return NULL; 334 return int_to_buf(buf.pointer); 335 } 336 337 static union acpi_object *acpi_label_read(acpi_handle handle, u32 offset, 338 u32 len) 339 { 340 acpi_status rc; 341 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 342 struct acpi_object_list input = { 343 .count = 2, 344 .pointer = (union acpi_object []) { 345 [0] = { 346 .integer.type = ACPI_TYPE_INTEGER, 347 .integer.value = offset, 348 }, 349 [1] = { 350 .integer.type = ACPI_TYPE_INTEGER, 351 .integer.value = len, 352 }, 353 }, 354 }; 355 356 rc = acpi_evaluate_object(handle, "_LSR", &input, &buf); 357 if (ACPI_FAILURE(rc)) 358 return NULL; 359 return pkg_to_buf(buf.pointer); 360 } 361 362 static union acpi_object *acpi_label_info(acpi_handle handle) 363 { 364 acpi_status rc; 365 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 366 367 rc = acpi_evaluate_object(handle, "_LSI", NULL, &buf); 368 if (ACPI_FAILURE(rc)) 369 return NULL; 370 return pkg_to_buf(buf.pointer); 371 } 372 373 static u8 nfit_dsm_revid(unsigned family, unsigned func) 374 { 375 static const u8 revid_table[NVDIMM_FAMILY_MAX+1][32] = { 376 [NVDIMM_FAMILY_INTEL] = { 377 [NVDIMM_INTEL_GET_MODES] = 2, 378 [NVDIMM_INTEL_GET_FWINFO] = 2, 379 [NVDIMM_INTEL_START_FWUPDATE] = 2, 380 [NVDIMM_INTEL_SEND_FWUPDATE] = 2, 381 [NVDIMM_INTEL_FINISH_FWUPDATE] = 2, 382 [NVDIMM_INTEL_QUERY_FWUPDATE] = 2, 383 [NVDIMM_INTEL_SET_THRESHOLD] = 2, 384 [NVDIMM_INTEL_INJECT_ERROR] = 2, 385 }, 386 }; 387 u8 id; 388 389 if (family > NVDIMM_FAMILY_MAX) 390 return 0; 391 if (func > 31) 392 return 0; 393 id = revid_table[family][func]; 394 if (id == 0) 395 return 1; /* default */ 396 return id; 397 } 398 399 int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm, 400 unsigned int cmd, void *buf, unsigned int buf_len, int *cmd_rc) 401 { 402 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 403 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 404 union acpi_object in_obj, in_buf, *out_obj; 405 const struct nd_cmd_desc *desc = NULL; 406 struct device *dev = acpi_desc->dev; 407 struct nd_cmd_pkg *call_pkg = NULL; 408 const char *cmd_name, *dimm_name; 409 unsigned long cmd_mask, dsm_mask; 410 u32 offset, fw_status = 0; 411 acpi_handle handle; 412 unsigned int func; 413 const guid_t *guid; 414 int rc, i; 415 416 func = cmd; 417 if (cmd == ND_CMD_CALL) { 418 call_pkg = buf; 419 func = call_pkg->nd_command; 420 421 for (i = 0; i < ARRAY_SIZE(call_pkg->nd_reserved2); i++) 422 if (call_pkg->nd_reserved2[i]) 423 return -EINVAL; 424 } 425 426 if (nvdimm) { 427 struct acpi_device *adev = nfit_mem->adev; 428 429 if (!adev) 430 return -ENOTTY; 431 if (call_pkg && nfit_mem->family != call_pkg->nd_family) 432 return -ENOTTY; 433 434 dimm_name = nvdimm_name(nvdimm); 435 cmd_name = nvdimm_cmd_name(cmd); 436 cmd_mask = nvdimm_cmd_mask(nvdimm); 437 dsm_mask = nfit_mem->dsm_mask; 438 desc = nd_cmd_dimm_desc(cmd); 439 guid = to_nfit_uuid(nfit_mem->family); 440 handle = adev->handle; 441 } else { 442 struct acpi_device *adev = to_acpi_dev(acpi_desc); 443 444 cmd_name = nvdimm_bus_cmd_name(cmd); 445 cmd_mask = nd_desc->cmd_mask; 446 dsm_mask = cmd_mask; 447 if (cmd == ND_CMD_CALL) 448 dsm_mask = nd_desc->bus_dsm_mask; 449 desc = nd_cmd_bus_desc(cmd); 450 guid = to_nfit_uuid(NFIT_DEV_BUS); 451 handle = adev->handle; 452 dimm_name = "bus"; 453 } 454 455 if (!desc || (cmd && (desc->out_num + desc->in_num == 0))) 456 return -ENOTTY; 457 458 if (!test_bit(cmd, &cmd_mask) || !test_bit(func, &dsm_mask)) 459 return -ENOTTY; 460 461 in_obj.type = ACPI_TYPE_PACKAGE; 462 in_obj.package.count = 1; 463 in_obj.package.elements = &in_buf; 464 in_buf.type = ACPI_TYPE_BUFFER; 465 in_buf.buffer.pointer = buf; 466 in_buf.buffer.length = 0; 467 468 /* libnvdimm has already validated the input envelope */ 469 for (i = 0; i < desc->in_num; i++) 470 in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc, 471 i, buf); 472 473 if (call_pkg) { 474 /* skip over package wrapper */ 475 in_buf.buffer.pointer = (void *) &call_pkg->nd_payload; 476 in_buf.buffer.length = call_pkg->nd_size_in; 477 } 478 479 dev_dbg(dev, "%s:%s cmd: %d: func: %d input length: %d\n", 480 __func__, dimm_name, cmd, func, in_buf.buffer.length); 481 print_hex_dump_debug("nvdimm in ", DUMP_PREFIX_OFFSET, 4, 4, 482 in_buf.buffer.pointer, 483 min_t(u32, 256, in_buf.buffer.length), true); 484 485 /* call the BIOS, prefer the named methods over _DSM if available */ 486 if (nvdimm && cmd == ND_CMD_GET_CONFIG_SIZE && nfit_mem->has_lsi) 487 out_obj = acpi_label_info(handle); 488 else if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && nfit_mem->has_lsr) { 489 struct nd_cmd_get_config_data_hdr *p = buf; 490 491 out_obj = acpi_label_read(handle, p->in_offset, p->in_length); 492 } else if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA 493 && nfit_mem->has_lsw) { 494 struct nd_cmd_set_config_hdr *p = buf; 495 496 out_obj = acpi_label_write(handle, p->in_offset, p->in_length, 497 p->in_buf); 498 } else { 499 u8 revid; 500 501 if (nvdimm) 502 revid = nfit_dsm_revid(nfit_mem->family, func); 503 else 504 revid = 1; 505 out_obj = acpi_evaluate_dsm(handle, guid, revid, func, &in_obj); 506 } 507 508 if (!out_obj) { 509 dev_dbg(dev, "%s:%s _DSM failed cmd: %s\n", __func__, dimm_name, 510 cmd_name); 511 return -EINVAL; 512 } 513 514 if (call_pkg) { 515 call_pkg->nd_fw_size = out_obj->buffer.length; 516 memcpy(call_pkg->nd_payload + call_pkg->nd_size_in, 517 out_obj->buffer.pointer, 518 min(call_pkg->nd_fw_size, call_pkg->nd_size_out)); 519 520 ACPI_FREE(out_obj); 521 /* 522 * Need to support FW function w/o known size in advance. 523 * Caller can determine required size based upon nd_fw_size. 524 * If we return an error (like elsewhere) then caller wouldn't 525 * be able to rely upon data returned to make calculation. 526 */ 527 return 0; 528 } 529 530 if (out_obj->package.type != ACPI_TYPE_BUFFER) { 531 dev_dbg(dev, "%s:%s unexpected output object type cmd: %s type: %d\n", 532 __func__, dimm_name, cmd_name, out_obj->type); 533 rc = -EINVAL; 534 goto out; 535 } 536 537 dev_dbg(dev, "%s:%s cmd: %s output length: %d\n", __func__, dimm_name, 538 cmd_name, out_obj->buffer.length); 539 print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4, 4, 540 out_obj->buffer.pointer, 541 min_t(u32, 128, out_obj->buffer.length), true); 542 543 for (i = 0, offset = 0; i < desc->out_num; i++) { 544 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf, 545 (u32 *) out_obj->buffer.pointer, 546 out_obj->buffer.length - offset); 547 548 if (offset + out_size > out_obj->buffer.length) { 549 dev_dbg(dev, "%s:%s output object underflow cmd: %s field: %d\n", 550 __func__, dimm_name, cmd_name, i); 551 break; 552 } 553 554 if (in_buf.buffer.length + offset + out_size > buf_len) { 555 dev_dbg(dev, "%s:%s output overrun cmd: %s field: %d\n", 556 __func__, dimm_name, cmd_name, i); 557 rc = -ENXIO; 558 goto out; 559 } 560 memcpy(buf + in_buf.buffer.length + offset, 561 out_obj->buffer.pointer + offset, out_size); 562 offset += out_size; 563 } 564 565 /* 566 * Set fw_status for all the commands with a known format to be 567 * later interpreted by xlat_status(). 568 */ 569 if (i >= 1 && ((!nvdimm && cmd >= ND_CMD_ARS_CAP 570 && cmd <= ND_CMD_CLEAR_ERROR) 571 || (nvdimm && cmd >= ND_CMD_SMART 572 && cmd <= ND_CMD_VENDOR))) 573 fw_status = *(u32 *) out_obj->buffer.pointer; 574 575 if (offset + in_buf.buffer.length < buf_len) { 576 if (i >= 1) { 577 /* 578 * status valid, return the number of bytes left 579 * unfilled in the output buffer 580 */ 581 rc = buf_len - offset - in_buf.buffer.length; 582 if (cmd_rc) 583 *cmd_rc = xlat_status(nvdimm, buf, cmd, 584 fw_status); 585 } else { 586 dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n", 587 __func__, dimm_name, cmd_name, buf_len, 588 offset); 589 rc = -ENXIO; 590 } 591 } else { 592 rc = 0; 593 if (cmd_rc) 594 *cmd_rc = xlat_status(nvdimm, buf, cmd, fw_status); 595 } 596 597 out: 598 ACPI_FREE(out_obj); 599 600 return rc; 601 } 602 EXPORT_SYMBOL_GPL(acpi_nfit_ctl); 603 604 static const char *spa_type_name(u16 type) 605 { 606 static const char *to_name[] = { 607 [NFIT_SPA_VOLATILE] = "volatile", 608 [NFIT_SPA_PM] = "pmem", 609 [NFIT_SPA_DCR] = "dimm-control-region", 610 [NFIT_SPA_BDW] = "block-data-window", 611 [NFIT_SPA_VDISK] = "volatile-disk", 612 [NFIT_SPA_VCD] = "volatile-cd", 613 [NFIT_SPA_PDISK] = "persistent-disk", 614 [NFIT_SPA_PCD] = "persistent-cd", 615 616 }; 617 618 if (type > NFIT_SPA_PCD) 619 return "unknown"; 620 621 return to_name[type]; 622 } 623 624 int nfit_spa_type(struct acpi_nfit_system_address *spa) 625 { 626 int i; 627 628 for (i = 0; i < NFIT_UUID_MAX; i++) 629 if (guid_equal(to_nfit_uuid(i), (guid_t *)&spa->range_guid)) 630 return i; 631 return -1; 632 } 633 634 static bool add_spa(struct acpi_nfit_desc *acpi_desc, 635 struct nfit_table_prev *prev, 636 struct acpi_nfit_system_address *spa) 637 { 638 struct device *dev = acpi_desc->dev; 639 struct nfit_spa *nfit_spa; 640 641 if (spa->header.length != sizeof(*spa)) 642 return false; 643 644 list_for_each_entry(nfit_spa, &prev->spas, list) { 645 if (memcmp(nfit_spa->spa, spa, sizeof(*spa)) == 0) { 646 list_move_tail(&nfit_spa->list, &acpi_desc->spas); 647 return true; 648 } 649 } 650 651 nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa) + sizeof(*spa), 652 GFP_KERNEL); 653 if (!nfit_spa) 654 return false; 655 INIT_LIST_HEAD(&nfit_spa->list); 656 memcpy(nfit_spa->spa, spa, sizeof(*spa)); 657 list_add_tail(&nfit_spa->list, &acpi_desc->spas); 658 dev_dbg(dev, "%s: spa index: %d type: %s\n", __func__, 659 spa->range_index, 660 spa_type_name(nfit_spa_type(spa))); 661 return true; 662 } 663 664 static bool add_memdev(struct acpi_nfit_desc *acpi_desc, 665 struct nfit_table_prev *prev, 666 struct acpi_nfit_memory_map *memdev) 667 { 668 struct device *dev = acpi_desc->dev; 669 struct nfit_memdev *nfit_memdev; 670 671 if (memdev->header.length != sizeof(*memdev)) 672 return false; 673 674 list_for_each_entry(nfit_memdev, &prev->memdevs, list) 675 if (memcmp(nfit_memdev->memdev, memdev, sizeof(*memdev)) == 0) { 676 list_move_tail(&nfit_memdev->list, &acpi_desc->memdevs); 677 return true; 678 } 679 680 nfit_memdev = devm_kzalloc(dev, sizeof(*nfit_memdev) + sizeof(*memdev), 681 GFP_KERNEL); 682 if (!nfit_memdev) 683 return false; 684 INIT_LIST_HEAD(&nfit_memdev->list); 685 memcpy(nfit_memdev->memdev, memdev, sizeof(*memdev)); 686 list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs); 687 dev_dbg(dev, "%s: memdev handle: %#x spa: %d dcr: %d flags: %#x\n", 688 __func__, memdev->device_handle, memdev->range_index, 689 memdev->region_index, memdev->flags); 690 return true; 691 } 692 693 /* 694 * An implementation may provide a truncated control region if no block windows 695 * are defined. 696 */ 697 static size_t sizeof_dcr(struct acpi_nfit_control_region *dcr) 698 { 699 if (dcr->header.length < offsetof(struct acpi_nfit_control_region, 700 window_size)) 701 return 0; 702 if (dcr->windows) 703 return sizeof(*dcr); 704 return offsetof(struct acpi_nfit_control_region, window_size); 705 } 706 707 static bool add_dcr(struct acpi_nfit_desc *acpi_desc, 708 struct nfit_table_prev *prev, 709 struct acpi_nfit_control_region *dcr) 710 { 711 struct device *dev = acpi_desc->dev; 712 struct nfit_dcr *nfit_dcr; 713 714 if (!sizeof_dcr(dcr)) 715 return false; 716 717 list_for_each_entry(nfit_dcr, &prev->dcrs, list) 718 if (memcmp(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)) == 0) { 719 list_move_tail(&nfit_dcr->list, &acpi_desc->dcrs); 720 return true; 721 } 722 723 nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr) + sizeof(*dcr), 724 GFP_KERNEL); 725 if (!nfit_dcr) 726 return false; 727 INIT_LIST_HEAD(&nfit_dcr->list); 728 memcpy(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)); 729 list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs); 730 dev_dbg(dev, "%s: dcr index: %d windows: %d\n", __func__, 731 dcr->region_index, dcr->windows); 732 return true; 733 } 734 735 static bool add_bdw(struct acpi_nfit_desc *acpi_desc, 736 struct nfit_table_prev *prev, 737 struct acpi_nfit_data_region *bdw) 738 { 739 struct device *dev = acpi_desc->dev; 740 struct nfit_bdw *nfit_bdw; 741 742 if (bdw->header.length != sizeof(*bdw)) 743 return false; 744 list_for_each_entry(nfit_bdw, &prev->bdws, list) 745 if (memcmp(nfit_bdw->bdw, bdw, sizeof(*bdw)) == 0) { 746 list_move_tail(&nfit_bdw->list, &acpi_desc->bdws); 747 return true; 748 } 749 750 nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw) + sizeof(*bdw), 751 GFP_KERNEL); 752 if (!nfit_bdw) 753 return false; 754 INIT_LIST_HEAD(&nfit_bdw->list); 755 memcpy(nfit_bdw->bdw, bdw, sizeof(*bdw)); 756 list_add_tail(&nfit_bdw->list, &acpi_desc->bdws); 757 dev_dbg(dev, "%s: bdw dcr: %d windows: %d\n", __func__, 758 bdw->region_index, bdw->windows); 759 return true; 760 } 761 762 static size_t sizeof_idt(struct acpi_nfit_interleave *idt) 763 { 764 if (idt->header.length < sizeof(*idt)) 765 return 0; 766 return sizeof(*idt) + sizeof(u32) * (idt->line_count - 1); 767 } 768 769 static bool add_idt(struct acpi_nfit_desc *acpi_desc, 770 struct nfit_table_prev *prev, 771 struct acpi_nfit_interleave *idt) 772 { 773 struct device *dev = acpi_desc->dev; 774 struct nfit_idt *nfit_idt; 775 776 if (!sizeof_idt(idt)) 777 return false; 778 779 list_for_each_entry(nfit_idt, &prev->idts, list) { 780 if (sizeof_idt(nfit_idt->idt) != sizeof_idt(idt)) 781 continue; 782 783 if (memcmp(nfit_idt->idt, idt, sizeof_idt(idt)) == 0) { 784 list_move_tail(&nfit_idt->list, &acpi_desc->idts); 785 return true; 786 } 787 } 788 789 nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt) + sizeof_idt(idt), 790 GFP_KERNEL); 791 if (!nfit_idt) 792 return false; 793 INIT_LIST_HEAD(&nfit_idt->list); 794 memcpy(nfit_idt->idt, idt, sizeof_idt(idt)); 795 list_add_tail(&nfit_idt->list, &acpi_desc->idts); 796 dev_dbg(dev, "%s: idt index: %d num_lines: %d\n", __func__, 797 idt->interleave_index, idt->line_count); 798 return true; 799 } 800 801 static size_t sizeof_flush(struct acpi_nfit_flush_address *flush) 802 { 803 if (flush->header.length < sizeof(*flush)) 804 return 0; 805 return sizeof(*flush) + sizeof(u64) * (flush->hint_count - 1); 806 } 807 808 static bool add_flush(struct acpi_nfit_desc *acpi_desc, 809 struct nfit_table_prev *prev, 810 struct acpi_nfit_flush_address *flush) 811 { 812 struct device *dev = acpi_desc->dev; 813 struct nfit_flush *nfit_flush; 814 815 if (!sizeof_flush(flush)) 816 return false; 817 818 list_for_each_entry(nfit_flush, &prev->flushes, list) { 819 if (sizeof_flush(nfit_flush->flush) != sizeof_flush(flush)) 820 continue; 821 822 if (memcmp(nfit_flush->flush, flush, 823 sizeof_flush(flush)) == 0) { 824 list_move_tail(&nfit_flush->list, &acpi_desc->flushes); 825 return true; 826 } 827 } 828 829 nfit_flush = devm_kzalloc(dev, sizeof(*nfit_flush) 830 + sizeof_flush(flush), GFP_KERNEL); 831 if (!nfit_flush) 832 return false; 833 INIT_LIST_HEAD(&nfit_flush->list); 834 memcpy(nfit_flush->flush, flush, sizeof_flush(flush)); 835 list_add_tail(&nfit_flush->list, &acpi_desc->flushes); 836 dev_dbg(dev, "%s: nfit_flush handle: %d hint_count: %d\n", __func__, 837 flush->device_handle, flush->hint_count); 838 return true; 839 } 840 841 static void *add_table(struct acpi_nfit_desc *acpi_desc, 842 struct nfit_table_prev *prev, void *table, const void *end) 843 { 844 struct device *dev = acpi_desc->dev; 845 struct acpi_nfit_header *hdr; 846 void *err = ERR_PTR(-ENOMEM); 847 848 if (table >= end) 849 return NULL; 850 851 hdr = table; 852 if (!hdr->length) { 853 dev_warn(dev, "found a zero length table '%d' parsing nfit\n", 854 hdr->type); 855 return NULL; 856 } 857 858 switch (hdr->type) { 859 case ACPI_NFIT_TYPE_SYSTEM_ADDRESS: 860 if (!add_spa(acpi_desc, prev, table)) 861 return err; 862 break; 863 case ACPI_NFIT_TYPE_MEMORY_MAP: 864 if (!add_memdev(acpi_desc, prev, table)) 865 return err; 866 break; 867 case ACPI_NFIT_TYPE_CONTROL_REGION: 868 if (!add_dcr(acpi_desc, prev, table)) 869 return err; 870 break; 871 case ACPI_NFIT_TYPE_DATA_REGION: 872 if (!add_bdw(acpi_desc, prev, table)) 873 return err; 874 break; 875 case ACPI_NFIT_TYPE_INTERLEAVE: 876 if (!add_idt(acpi_desc, prev, table)) 877 return err; 878 break; 879 case ACPI_NFIT_TYPE_FLUSH_ADDRESS: 880 if (!add_flush(acpi_desc, prev, table)) 881 return err; 882 break; 883 case ACPI_NFIT_TYPE_SMBIOS: 884 dev_dbg(dev, "%s: smbios\n", __func__); 885 break; 886 default: 887 dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type); 888 break; 889 } 890 891 return table + hdr->length; 892 } 893 894 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc, 895 struct nfit_mem *nfit_mem) 896 { 897 u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle; 898 u16 dcr = nfit_mem->dcr->region_index; 899 struct nfit_spa *nfit_spa; 900 901 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 902 u16 range_index = nfit_spa->spa->range_index; 903 int type = nfit_spa_type(nfit_spa->spa); 904 struct nfit_memdev *nfit_memdev; 905 906 if (type != NFIT_SPA_BDW) 907 continue; 908 909 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 910 if (nfit_memdev->memdev->range_index != range_index) 911 continue; 912 if (nfit_memdev->memdev->device_handle != device_handle) 913 continue; 914 if (nfit_memdev->memdev->region_index != dcr) 915 continue; 916 917 nfit_mem->spa_bdw = nfit_spa->spa; 918 return; 919 } 920 } 921 922 dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n", 923 nfit_mem->spa_dcr->range_index); 924 nfit_mem->bdw = NULL; 925 } 926 927 static void nfit_mem_init_bdw(struct acpi_nfit_desc *acpi_desc, 928 struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa) 929 { 930 u16 dcr = __to_nfit_memdev(nfit_mem)->region_index; 931 struct nfit_memdev *nfit_memdev; 932 struct nfit_bdw *nfit_bdw; 933 struct nfit_idt *nfit_idt; 934 u16 idt_idx, range_index; 935 936 list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) { 937 if (nfit_bdw->bdw->region_index != dcr) 938 continue; 939 nfit_mem->bdw = nfit_bdw->bdw; 940 break; 941 } 942 943 if (!nfit_mem->bdw) 944 return; 945 946 nfit_mem_find_spa_bdw(acpi_desc, nfit_mem); 947 948 if (!nfit_mem->spa_bdw) 949 return; 950 951 range_index = nfit_mem->spa_bdw->range_index; 952 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 953 if (nfit_memdev->memdev->range_index != range_index || 954 nfit_memdev->memdev->region_index != dcr) 955 continue; 956 nfit_mem->memdev_bdw = nfit_memdev->memdev; 957 idt_idx = nfit_memdev->memdev->interleave_index; 958 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) { 959 if (nfit_idt->idt->interleave_index != idt_idx) 960 continue; 961 nfit_mem->idt_bdw = nfit_idt->idt; 962 break; 963 } 964 break; 965 } 966 } 967 968 static int __nfit_mem_init(struct acpi_nfit_desc *acpi_desc, 969 struct acpi_nfit_system_address *spa) 970 { 971 struct nfit_mem *nfit_mem, *found; 972 struct nfit_memdev *nfit_memdev; 973 int type = spa ? nfit_spa_type(spa) : 0; 974 975 switch (type) { 976 case NFIT_SPA_DCR: 977 case NFIT_SPA_PM: 978 break; 979 default: 980 if (spa) 981 return 0; 982 } 983 984 /* 985 * This loop runs in two modes, when a dimm is mapped the loop 986 * adds memdev associations to an existing dimm, or creates a 987 * dimm. In the unmapped dimm case this loop sweeps for memdev 988 * instances with an invalid / zero range_index and adds those 989 * dimms without spa associations. 990 */ 991 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 992 struct nfit_flush *nfit_flush; 993 struct nfit_dcr *nfit_dcr; 994 u32 device_handle; 995 u16 dcr; 996 997 if (spa && nfit_memdev->memdev->range_index != spa->range_index) 998 continue; 999 if (!spa && nfit_memdev->memdev->range_index) 1000 continue; 1001 found = NULL; 1002 dcr = nfit_memdev->memdev->region_index; 1003 device_handle = nfit_memdev->memdev->device_handle; 1004 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) 1005 if (__to_nfit_memdev(nfit_mem)->device_handle 1006 == device_handle) { 1007 found = nfit_mem; 1008 break; 1009 } 1010 1011 if (found) 1012 nfit_mem = found; 1013 else { 1014 nfit_mem = devm_kzalloc(acpi_desc->dev, 1015 sizeof(*nfit_mem), GFP_KERNEL); 1016 if (!nfit_mem) 1017 return -ENOMEM; 1018 INIT_LIST_HEAD(&nfit_mem->list); 1019 nfit_mem->acpi_desc = acpi_desc; 1020 list_add(&nfit_mem->list, &acpi_desc->dimms); 1021 } 1022 1023 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) { 1024 if (nfit_dcr->dcr->region_index != dcr) 1025 continue; 1026 /* 1027 * Record the control region for the dimm. For 1028 * the ACPI 6.1 case, where there are separate 1029 * control regions for the pmem vs blk 1030 * interfaces, be sure to record the extended 1031 * blk details. 1032 */ 1033 if (!nfit_mem->dcr) 1034 nfit_mem->dcr = nfit_dcr->dcr; 1035 else if (nfit_mem->dcr->windows == 0 1036 && nfit_dcr->dcr->windows) 1037 nfit_mem->dcr = nfit_dcr->dcr; 1038 break; 1039 } 1040 1041 list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) { 1042 struct acpi_nfit_flush_address *flush; 1043 u16 i; 1044 1045 if (nfit_flush->flush->device_handle != device_handle) 1046 continue; 1047 nfit_mem->nfit_flush = nfit_flush; 1048 flush = nfit_flush->flush; 1049 nfit_mem->flush_wpq = devm_kzalloc(acpi_desc->dev, 1050 flush->hint_count 1051 * sizeof(struct resource), GFP_KERNEL); 1052 if (!nfit_mem->flush_wpq) 1053 return -ENOMEM; 1054 for (i = 0; i < flush->hint_count; i++) { 1055 struct resource *res = &nfit_mem->flush_wpq[i]; 1056 1057 res->start = flush->hint_address[i]; 1058 res->end = res->start + 8 - 1; 1059 } 1060 break; 1061 } 1062 1063 if (dcr && !nfit_mem->dcr) { 1064 dev_err(acpi_desc->dev, "SPA %d missing DCR %d\n", 1065 spa->range_index, dcr); 1066 return -ENODEV; 1067 } 1068 1069 if (type == NFIT_SPA_DCR) { 1070 struct nfit_idt *nfit_idt; 1071 u16 idt_idx; 1072 1073 /* multiple dimms may share a SPA when interleaved */ 1074 nfit_mem->spa_dcr = spa; 1075 nfit_mem->memdev_dcr = nfit_memdev->memdev; 1076 idt_idx = nfit_memdev->memdev->interleave_index; 1077 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) { 1078 if (nfit_idt->idt->interleave_index != idt_idx) 1079 continue; 1080 nfit_mem->idt_dcr = nfit_idt->idt; 1081 break; 1082 } 1083 nfit_mem_init_bdw(acpi_desc, nfit_mem, spa); 1084 } else if (type == NFIT_SPA_PM) { 1085 /* 1086 * A single dimm may belong to multiple SPA-PM 1087 * ranges, record at least one in addition to 1088 * any SPA-DCR range. 1089 */ 1090 nfit_mem->memdev_pmem = nfit_memdev->memdev; 1091 } else 1092 nfit_mem->memdev_dcr = nfit_memdev->memdev; 1093 } 1094 1095 return 0; 1096 } 1097 1098 static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b) 1099 { 1100 struct nfit_mem *a = container_of(_a, typeof(*a), list); 1101 struct nfit_mem *b = container_of(_b, typeof(*b), list); 1102 u32 handleA, handleB; 1103 1104 handleA = __to_nfit_memdev(a)->device_handle; 1105 handleB = __to_nfit_memdev(b)->device_handle; 1106 if (handleA < handleB) 1107 return -1; 1108 else if (handleA > handleB) 1109 return 1; 1110 return 0; 1111 } 1112 1113 static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc) 1114 { 1115 struct nfit_spa *nfit_spa; 1116 int rc; 1117 1118 1119 /* 1120 * For each SPA-DCR or SPA-PMEM address range find its 1121 * corresponding MEMDEV(s). From each MEMDEV find the 1122 * corresponding DCR. Then, if we're operating on a SPA-DCR, 1123 * try to find a SPA-BDW and a corresponding BDW that references 1124 * the DCR. Throw it all into an nfit_mem object. Note, that 1125 * BDWs are optional. 1126 */ 1127 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 1128 rc = __nfit_mem_init(acpi_desc, nfit_spa->spa); 1129 if (rc) 1130 return rc; 1131 } 1132 1133 /* 1134 * If a DIMM has failed to be mapped into SPA there will be no 1135 * SPA entries above. Find and register all the unmapped DIMMs 1136 * for reporting and recovery purposes. 1137 */ 1138 rc = __nfit_mem_init(acpi_desc, NULL); 1139 if (rc) 1140 return rc; 1141 1142 list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp); 1143 1144 return 0; 1145 } 1146 1147 static ssize_t bus_dsm_mask_show(struct device *dev, 1148 struct device_attribute *attr, char *buf) 1149 { 1150 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1151 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1152 1153 return sprintf(buf, "%#lx\n", nd_desc->bus_dsm_mask); 1154 } 1155 static struct device_attribute dev_attr_bus_dsm_mask = 1156 __ATTR(dsm_mask, 0444, bus_dsm_mask_show, NULL); 1157 1158 static ssize_t revision_show(struct device *dev, 1159 struct device_attribute *attr, char *buf) 1160 { 1161 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1162 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1163 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1164 1165 return sprintf(buf, "%d\n", acpi_desc->acpi_header.revision); 1166 } 1167 static DEVICE_ATTR_RO(revision); 1168 1169 static ssize_t hw_error_scrub_show(struct device *dev, 1170 struct device_attribute *attr, char *buf) 1171 { 1172 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1173 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1174 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1175 1176 return sprintf(buf, "%d\n", acpi_desc->scrub_mode); 1177 } 1178 1179 /* 1180 * The 'hw_error_scrub' attribute can have the following values written to it: 1181 * '0': Switch to the default mode where an exception will only insert 1182 * the address of the memory error into the poison and badblocks lists. 1183 * '1': Enable a full scrub to happen if an exception for a memory error is 1184 * received. 1185 */ 1186 static ssize_t hw_error_scrub_store(struct device *dev, 1187 struct device_attribute *attr, const char *buf, size_t size) 1188 { 1189 struct nvdimm_bus_descriptor *nd_desc; 1190 ssize_t rc; 1191 long val; 1192 1193 rc = kstrtol(buf, 0, &val); 1194 if (rc) 1195 return rc; 1196 1197 device_lock(dev); 1198 nd_desc = dev_get_drvdata(dev); 1199 if (nd_desc) { 1200 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1201 1202 switch (val) { 1203 case HW_ERROR_SCRUB_ON: 1204 acpi_desc->scrub_mode = HW_ERROR_SCRUB_ON; 1205 break; 1206 case HW_ERROR_SCRUB_OFF: 1207 acpi_desc->scrub_mode = HW_ERROR_SCRUB_OFF; 1208 break; 1209 default: 1210 rc = -EINVAL; 1211 break; 1212 } 1213 } 1214 device_unlock(dev); 1215 if (rc) 1216 return rc; 1217 return size; 1218 } 1219 static DEVICE_ATTR_RW(hw_error_scrub); 1220 1221 /* 1222 * This shows the number of full Address Range Scrubs that have been 1223 * completed since driver load time. Userspace can wait on this using 1224 * select/poll etc. A '+' at the end indicates an ARS is in progress 1225 */ 1226 static ssize_t scrub_show(struct device *dev, 1227 struct device_attribute *attr, char *buf) 1228 { 1229 struct nvdimm_bus_descriptor *nd_desc; 1230 ssize_t rc = -ENXIO; 1231 1232 device_lock(dev); 1233 nd_desc = dev_get_drvdata(dev); 1234 if (nd_desc) { 1235 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1236 1237 rc = sprintf(buf, "%d%s", acpi_desc->scrub_count, 1238 (work_busy(&acpi_desc->work)) ? "+\n" : "\n"); 1239 } 1240 device_unlock(dev); 1241 return rc; 1242 } 1243 1244 static ssize_t scrub_store(struct device *dev, 1245 struct device_attribute *attr, const char *buf, size_t size) 1246 { 1247 struct nvdimm_bus_descriptor *nd_desc; 1248 ssize_t rc; 1249 long val; 1250 1251 rc = kstrtol(buf, 0, &val); 1252 if (rc) 1253 return rc; 1254 if (val != 1) 1255 return -EINVAL; 1256 1257 device_lock(dev); 1258 nd_desc = dev_get_drvdata(dev); 1259 if (nd_desc) { 1260 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1261 1262 rc = acpi_nfit_ars_rescan(acpi_desc, 0); 1263 } 1264 device_unlock(dev); 1265 if (rc) 1266 return rc; 1267 return size; 1268 } 1269 static DEVICE_ATTR_RW(scrub); 1270 1271 static bool ars_supported(struct nvdimm_bus *nvdimm_bus) 1272 { 1273 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1274 const unsigned long mask = 1 << ND_CMD_ARS_CAP | 1 << ND_CMD_ARS_START 1275 | 1 << ND_CMD_ARS_STATUS; 1276 1277 return (nd_desc->cmd_mask & mask) == mask; 1278 } 1279 1280 static umode_t nfit_visible(struct kobject *kobj, struct attribute *a, int n) 1281 { 1282 struct device *dev = container_of(kobj, struct device, kobj); 1283 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1284 1285 if (a == &dev_attr_scrub.attr && !ars_supported(nvdimm_bus)) 1286 return 0; 1287 return a->mode; 1288 } 1289 1290 static struct attribute *acpi_nfit_attributes[] = { 1291 &dev_attr_revision.attr, 1292 &dev_attr_scrub.attr, 1293 &dev_attr_hw_error_scrub.attr, 1294 &dev_attr_bus_dsm_mask.attr, 1295 NULL, 1296 }; 1297 1298 static const struct attribute_group acpi_nfit_attribute_group = { 1299 .name = "nfit", 1300 .attrs = acpi_nfit_attributes, 1301 .is_visible = nfit_visible, 1302 }; 1303 1304 static const struct attribute_group *acpi_nfit_attribute_groups[] = { 1305 &nvdimm_bus_attribute_group, 1306 &acpi_nfit_attribute_group, 1307 NULL, 1308 }; 1309 1310 static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev) 1311 { 1312 struct nvdimm *nvdimm = to_nvdimm(dev); 1313 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1314 1315 return __to_nfit_memdev(nfit_mem); 1316 } 1317 1318 static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev) 1319 { 1320 struct nvdimm *nvdimm = to_nvdimm(dev); 1321 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1322 1323 return nfit_mem->dcr; 1324 } 1325 1326 static ssize_t handle_show(struct device *dev, 1327 struct device_attribute *attr, char *buf) 1328 { 1329 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev); 1330 1331 return sprintf(buf, "%#x\n", memdev->device_handle); 1332 } 1333 static DEVICE_ATTR_RO(handle); 1334 1335 static ssize_t phys_id_show(struct device *dev, 1336 struct device_attribute *attr, char *buf) 1337 { 1338 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev); 1339 1340 return sprintf(buf, "%#x\n", memdev->physical_id); 1341 } 1342 static DEVICE_ATTR_RO(phys_id); 1343 1344 static ssize_t vendor_show(struct device *dev, 1345 struct device_attribute *attr, char *buf) 1346 { 1347 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1348 1349 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->vendor_id)); 1350 } 1351 static DEVICE_ATTR_RO(vendor); 1352 1353 static ssize_t rev_id_show(struct device *dev, 1354 struct device_attribute *attr, char *buf) 1355 { 1356 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1357 1358 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->revision_id)); 1359 } 1360 static DEVICE_ATTR_RO(rev_id); 1361 1362 static ssize_t device_show(struct device *dev, 1363 struct device_attribute *attr, char *buf) 1364 { 1365 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1366 1367 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->device_id)); 1368 } 1369 static DEVICE_ATTR_RO(device); 1370 1371 static ssize_t subsystem_vendor_show(struct device *dev, 1372 struct device_attribute *attr, char *buf) 1373 { 1374 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1375 1376 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_vendor_id)); 1377 } 1378 static DEVICE_ATTR_RO(subsystem_vendor); 1379 1380 static ssize_t subsystem_rev_id_show(struct device *dev, 1381 struct device_attribute *attr, char *buf) 1382 { 1383 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1384 1385 return sprintf(buf, "0x%04x\n", 1386 be16_to_cpu(dcr->subsystem_revision_id)); 1387 } 1388 static DEVICE_ATTR_RO(subsystem_rev_id); 1389 1390 static ssize_t subsystem_device_show(struct device *dev, 1391 struct device_attribute *attr, char *buf) 1392 { 1393 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1394 1395 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_device_id)); 1396 } 1397 static DEVICE_ATTR_RO(subsystem_device); 1398 1399 static int num_nvdimm_formats(struct nvdimm *nvdimm) 1400 { 1401 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1402 int formats = 0; 1403 1404 if (nfit_mem->memdev_pmem) 1405 formats++; 1406 if (nfit_mem->memdev_bdw) 1407 formats++; 1408 return formats; 1409 } 1410 1411 static ssize_t format_show(struct device *dev, 1412 struct device_attribute *attr, char *buf) 1413 { 1414 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1415 1416 return sprintf(buf, "0x%04x\n", le16_to_cpu(dcr->code)); 1417 } 1418 static DEVICE_ATTR_RO(format); 1419 1420 static ssize_t format1_show(struct device *dev, 1421 struct device_attribute *attr, char *buf) 1422 { 1423 u32 handle; 1424 ssize_t rc = -ENXIO; 1425 struct nfit_mem *nfit_mem; 1426 struct nfit_memdev *nfit_memdev; 1427 struct acpi_nfit_desc *acpi_desc; 1428 struct nvdimm *nvdimm = to_nvdimm(dev); 1429 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1430 1431 nfit_mem = nvdimm_provider_data(nvdimm); 1432 acpi_desc = nfit_mem->acpi_desc; 1433 handle = to_nfit_memdev(dev)->device_handle; 1434 1435 /* assumes DIMMs have at most 2 published interface codes */ 1436 mutex_lock(&acpi_desc->init_mutex); 1437 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 1438 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev; 1439 struct nfit_dcr *nfit_dcr; 1440 1441 if (memdev->device_handle != handle) 1442 continue; 1443 1444 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) { 1445 if (nfit_dcr->dcr->region_index != memdev->region_index) 1446 continue; 1447 if (nfit_dcr->dcr->code == dcr->code) 1448 continue; 1449 rc = sprintf(buf, "0x%04x\n", 1450 le16_to_cpu(nfit_dcr->dcr->code)); 1451 break; 1452 } 1453 if (rc != ENXIO) 1454 break; 1455 } 1456 mutex_unlock(&acpi_desc->init_mutex); 1457 return rc; 1458 } 1459 static DEVICE_ATTR_RO(format1); 1460 1461 static ssize_t formats_show(struct device *dev, 1462 struct device_attribute *attr, char *buf) 1463 { 1464 struct nvdimm *nvdimm = to_nvdimm(dev); 1465 1466 return sprintf(buf, "%d\n", num_nvdimm_formats(nvdimm)); 1467 } 1468 static DEVICE_ATTR_RO(formats); 1469 1470 static ssize_t serial_show(struct device *dev, 1471 struct device_attribute *attr, char *buf) 1472 { 1473 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1474 1475 return sprintf(buf, "0x%08x\n", be32_to_cpu(dcr->serial_number)); 1476 } 1477 static DEVICE_ATTR_RO(serial); 1478 1479 static ssize_t family_show(struct device *dev, 1480 struct device_attribute *attr, char *buf) 1481 { 1482 struct nvdimm *nvdimm = to_nvdimm(dev); 1483 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1484 1485 if (nfit_mem->family < 0) 1486 return -ENXIO; 1487 return sprintf(buf, "%d\n", nfit_mem->family); 1488 } 1489 static DEVICE_ATTR_RO(family); 1490 1491 static ssize_t dsm_mask_show(struct device *dev, 1492 struct device_attribute *attr, char *buf) 1493 { 1494 struct nvdimm *nvdimm = to_nvdimm(dev); 1495 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1496 1497 if (nfit_mem->family < 0) 1498 return -ENXIO; 1499 return sprintf(buf, "%#lx\n", nfit_mem->dsm_mask); 1500 } 1501 static DEVICE_ATTR_RO(dsm_mask); 1502 1503 static ssize_t flags_show(struct device *dev, 1504 struct device_attribute *attr, char *buf) 1505 { 1506 u16 flags = to_nfit_memdev(dev)->flags; 1507 1508 return sprintf(buf, "%s%s%s%s%s%s%s\n", 1509 flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "", 1510 flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "", 1511 flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "", 1512 flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "", 1513 flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : "", 1514 flags & ACPI_NFIT_MEM_MAP_FAILED ? "map_fail " : "", 1515 flags & ACPI_NFIT_MEM_HEALTH_ENABLED ? "smart_notify " : ""); 1516 } 1517 static DEVICE_ATTR_RO(flags); 1518 1519 static ssize_t id_show(struct device *dev, 1520 struct device_attribute *attr, char *buf) 1521 { 1522 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1523 1524 if (dcr->valid_fields & ACPI_NFIT_CONTROL_MFG_INFO_VALID) 1525 return sprintf(buf, "%04x-%02x-%04x-%08x\n", 1526 be16_to_cpu(dcr->vendor_id), 1527 dcr->manufacturing_location, 1528 be16_to_cpu(dcr->manufacturing_date), 1529 be32_to_cpu(dcr->serial_number)); 1530 else 1531 return sprintf(buf, "%04x-%08x\n", 1532 be16_to_cpu(dcr->vendor_id), 1533 be32_to_cpu(dcr->serial_number)); 1534 } 1535 static DEVICE_ATTR_RO(id); 1536 1537 static struct attribute *acpi_nfit_dimm_attributes[] = { 1538 &dev_attr_handle.attr, 1539 &dev_attr_phys_id.attr, 1540 &dev_attr_vendor.attr, 1541 &dev_attr_device.attr, 1542 &dev_attr_rev_id.attr, 1543 &dev_attr_subsystem_vendor.attr, 1544 &dev_attr_subsystem_device.attr, 1545 &dev_attr_subsystem_rev_id.attr, 1546 &dev_attr_format.attr, 1547 &dev_attr_formats.attr, 1548 &dev_attr_format1.attr, 1549 &dev_attr_serial.attr, 1550 &dev_attr_flags.attr, 1551 &dev_attr_id.attr, 1552 &dev_attr_family.attr, 1553 &dev_attr_dsm_mask.attr, 1554 NULL, 1555 }; 1556 1557 static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj, 1558 struct attribute *a, int n) 1559 { 1560 struct device *dev = container_of(kobj, struct device, kobj); 1561 struct nvdimm *nvdimm = to_nvdimm(dev); 1562 1563 if (!to_nfit_dcr(dev)) { 1564 /* Without a dcr only the memdev attributes can be surfaced */ 1565 if (a == &dev_attr_handle.attr || a == &dev_attr_phys_id.attr 1566 || a == &dev_attr_flags.attr 1567 || a == &dev_attr_family.attr 1568 || a == &dev_attr_dsm_mask.attr) 1569 return a->mode; 1570 return 0; 1571 } 1572 1573 if (a == &dev_attr_format1.attr && num_nvdimm_formats(nvdimm) <= 1) 1574 return 0; 1575 return a->mode; 1576 } 1577 1578 static const struct attribute_group acpi_nfit_dimm_attribute_group = { 1579 .name = "nfit", 1580 .attrs = acpi_nfit_dimm_attributes, 1581 .is_visible = acpi_nfit_dimm_attr_visible, 1582 }; 1583 1584 static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = { 1585 &nvdimm_attribute_group, 1586 &nd_device_attribute_group, 1587 &acpi_nfit_dimm_attribute_group, 1588 NULL, 1589 }; 1590 1591 static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc, 1592 u32 device_handle) 1593 { 1594 struct nfit_mem *nfit_mem; 1595 1596 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) 1597 if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle) 1598 return nfit_mem->nvdimm; 1599 1600 return NULL; 1601 } 1602 1603 void __acpi_nvdimm_notify(struct device *dev, u32 event) 1604 { 1605 struct nfit_mem *nfit_mem; 1606 struct acpi_nfit_desc *acpi_desc; 1607 1608 dev_dbg(dev->parent, "%s: %s: event: %d\n", dev_name(dev), __func__, 1609 event); 1610 1611 if (event != NFIT_NOTIFY_DIMM_HEALTH) { 1612 dev_dbg(dev->parent, "%s: unknown event: %d\n", dev_name(dev), 1613 event); 1614 return; 1615 } 1616 1617 acpi_desc = dev_get_drvdata(dev->parent); 1618 if (!acpi_desc) 1619 return; 1620 1621 /* 1622 * If we successfully retrieved acpi_desc, then we know nfit_mem data 1623 * is still valid. 1624 */ 1625 nfit_mem = dev_get_drvdata(dev); 1626 if (nfit_mem && nfit_mem->flags_attr) 1627 sysfs_notify_dirent(nfit_mem->flags_attr); 1628 } 1629 EXPORT_SYMBOL_GPL(__acpi_nvdimm_notify); 1630 1631 static void acpi_nvdimm_notify(acpi_handle handle, u32 event, void *data) 1632 { 1633 struct acpi_device *adev = data; 1634 struct device *dev = &adev->dev; 1635 1636 device_lock(dev->parent); 1637 __acpi_nvdimm_notify(dev, event); 1638 device_unlock(dev->parent); 1639 } 1640 1641 static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc, 1642 struct nfit_mem *nfit_mem, u32 device_handle) 1643 { 1644 struct acpi_device *adev, *adev_dimm; 1645 struct device *dev = acpi_desc->dev; 1646 union acpi_object *obj; 1647 unsigned long dsm_mask; 1648 const guid_t *guid; 1649 int i; 1650 int family = -1; 1651 1652 /* nfit test assumes 1:1 relationship between commands and dsms */ 1653 nfit_mem->dsm_mask = acpi_desc->dimm_cmd_force_en; 1654 nfit_mem->family = NVDIMM_FAMILY_INTEL; 1655 adev = to_acpi_dev(acpi_desc); 1656 if (!adev) 1657 return 0; 1658 1659 adev_dimm = acpi_find_child_device(adev, device_handle, false); 1660 nfit_mem->adev = adev_dimm; 1661 if (!adev_dimm) { 1662 dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n", 1663 device_handle); 1664 return force_enable_dimms ? 0 : -ENODEV; 1665 } 1666 1667 if (ACPI_FAILURE(acpi_install_notify_handler(adev_dimm->handle, 1668 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify, adev_dimm))) { 1669 dev_err(dev, "%s: notification registration failed\n", 1670 dev_name(&adev_dimm->dev)); 1671 return -ENXIO; 1672 } 1673 1674 /* 1675 * Until standardization materializes we need to consider 4 1676 * different command sets. Note, that checking for function0 (bit0) 1677 * tells us if any commands are reachable through this GUID. 1678 */ 1679 for (i = 0; i <= NVDIMM_FAMILY_MAX; i++) 1680 if (acpi_check_dsm(adev_dimm->handle, to_nfit_uuid(i), 1, 1)) 1681 if (family < 0 || i == default_dsm_family) 1682 family = i; 1683 1684 /* limit the supported commands to those that are publicly documented */ 1685 nfit_mem->family = family; 1686 if (override_dsm_mask && !disable_vendor_specific) 1687 dsm_mask = override_dsm_mask; 1688 else if (nfit_mem->family == NVDIMM_FAMILY_INTEL) { 1689 dsm_mask = NVDIMM_INTEL_CMDMASK; 1690 if (disable_vendor_specific) 1691 dsm_mask &= ~(1 << ND_CMD_VENDOR); 1692 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE1) { 1693 dsm_mask = 0x1c3c76; 1694 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE2) { 1695 dsm_mask = 0x1fe; 1696 if (disable_vendor_specific) 1697 dsm_mask &= ~(1 << 8); 1698 } else if (nfit_mem->family == NVDIMM_FAMILY_MSFT) { 1699 dsm_mask = 0xffffffff; 1700 } else { 1701 dev_dbg(dev, "unknown dimm command family\n"); 1702 nfit_mem->family = -1; 1703 /* DSMs are optional, continue loading the driver... */ 1704 return 0; 1705 } 1706 1707 guid = to_nfit_uuid(nfit_mem->family); 1708 for_each_set_bit(i, &dsm_mask, BITS_PER_LONG) 1709 if (acpi_check_dsm(adev_dimm->handle, guid, 1710 nfit_dsm_revid(nfit_mem->family, i), 1711 1ULL << i)) 1712 set_bit(i, &nfit_mem->dsm_mask); 1713 1714 obj = acpi_label_info(adev_dimm->handle); 1715 if (obj) { 1716 ACPI_FREE(obj); 1717 nfit_mem->has_lsi = 1; 1718 dev_dbg(dev, "%s: has _LSI\n", dev_name(&adev_dimm->dev)); 1719 } 1720 1721 obj = acpi_label_read(adev_dimm->handle, 0, 0); 1722 if (obj) { 1723 ACPI_FREE(obj); 1724 nfit_mem->has_lsr = 1; 1725 dev_dbg(dev, "%s: has _LSR\n", dev_name(&adev_dimm->dev)); 1726 } 1727 1728 obj = acpi_label_write(adev_dimm->handle, 0, 0, NULL); 1729 if (obj) { 1730 ACPI_FREE(obj); 1731 nfit_mem->has_lsw = 1; 1732 dev_dbg(dev, "%s: has _LSW\n", dev_name(&adev_dimm->dev)); 1733 } 1734 1735 return 0; 1736 } 1737 1738 static void shutdown_dimm_notify(void *data) 1739 { 1740 struct acpi_nfit_desc *acpi_desc = data; 1741 struct nfit_mem *nfit_mem; 1742 1743 mutex_lock(&acpi_desc->init_mutex); 1744 /* 1745 * Clear out the nfit_mem->flags_attr and shut down dimm event 1746 * notifications. 1747 */ 1748 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1749 struct acpi_device *adev_dimm = nfit_mem->adev; 1750 1751 if (nfit_mem->flags_attr) { 1752 sysfs_put(nfit_mem->flags_attr); 1753 nfit_mem->flags_attr = NULL; 1754 } 1755 if (adev_dimm) 1756 acpi_remove_notify_handler(adev_dimm->handle, 1757 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify); 1758 } 1759 mutex_unlock(&acpi_desc->init_mutex); 1760 } 1761 1762 static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc) 1763 { 1764 struct nfit_mem *nfit_mem; 1765 int dimm_count = 0, rc; 1766 struct nvdimm *nvdimm; 1767 1768 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1769 struct acpi_nfit_flush_address *flush; 1770 unsigned long flags = 0, cmd_mask; 1771 struct nfit_memdev *nfit_memdev; 1772 u32 device_handle; 1773 u16 mem_flags; 1774 1775 device_handle = __to_nfit_memdev(nfit_mem)->device_handle; 1776 nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle); 1777 if (nvdimm) { 1778 dimm_count++; 1779 continue; 1780 } 1781 1782 if (nfit_mem->bdw && nfit_mem->memdev_pmem) 1783 set_bit(NDD_ALIASING, &flags); 1784 1785 /* collate flags across all memdevs for this dimm */ 1786 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 1787 struct acpi_nfit_memory_map *dimm_memdev; 1788 1789 dimm_memdev = __to_nfit_memdev(nfit_mem); 1790 if (dimm_memdev->device_handle 1791 != nfit_memdev->memdev->device_handle) 1792 continue; 1793 dimm_memdev->flags |= nfit_memdev->memdev->flags; 1794 } 1795 1796 mem_flags = __to_nfit_memdev(nfit_mem)->flags; 1797 if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED) 1798 set_bit(NDD_UNARMED, &flags); 1799 1800 rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle); 1801 if (rc) 1802 continue; 1803 1804 /* 1805 * TODO: provide translation for non-NVDIMM_FAMILY_INTEL 1806 * devices (i.e. from nd_cmd to acpi_dsm) to standardize the 1807 * userspace interface. 1808 */ 1809 cmd_mask = 1UL << ND_CMD_CALL; 1810 if (nfit_mem->family == NVDIMM_FAMILY_INTEL) { 1811 /* 1812 * These commands have a 1:1 correspondence 1813 * between DSM payload and libnvdimm ioctl 1814 * payload format. 1815 */ 1816 cmd_mask |= nfit_mem->dsm_mask & NVDIMM_STANDARD_CMDMASK; 1817 } 1818 1819 if (nfit_mem->has_lsi) 1820 set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask); 1821 if (nfit_mem->has_lsr) 1822 set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask); 1823 if (nfit_mem->has_lsw) 1824 set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask); 1825 1826 flush = nfit_mem->nfit_flush ? nfit_mem->nfit_flush->flush 1827 : NULL; 1828 nvdimm = nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem, 1829 acpi_nfit_dimm_attribute_groups, 1830 flags, cmd_mask, flush ? flush->hint_count : 0, 1831 nfit_mem->flush_wpq); 1832 if (!nvdimm) 1833 return -ENOMEM; 1834 1835 nfit_mem->nvdimm = nvdimm; 1836 dimm_count++; 1837 1838 if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0) 1839 continue; 1840 1841 dev_info(acpi_desc->dev, "%s flags:%s%s%s%s%s\n", 1842 nvdimm_name(nvdimm), 1843 mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "", 1844 mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"", 1845 mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "", 1846 mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : "", 1847 mem_flags & ACPI_NFIT_MEM_MAP_FAILED ? " map_fail" : ""); 1848 1849 } 1850 1851 rc = nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count); 1852 if (rc) 1853 return rc; 1854 1855 /* 1856 * Now that dimms are successfully registered, and async registration 1857 * is flushed, attempt to enable event notification. 1858 */ 1859 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1860 struct kernfs_node *nfit_kernfs; 1861 1862 nvdimm = nfit_mem->nvdimm; 1863 nfit_kernfs = sysfs_get_dirent(nvdimm_kobj(nvdimm)->sd, "nfit"); 1864 if (nfit_kernfs) 1865 nfit_mem->flags_attr = sysfs_get_dirent(nfit_kernfs, 1866 "flags"); 1867 sysfs_put(nfit_kernfs); 1868 if (!nfit_mem->flags_attr) 1869 dev_warn(acpi_desc->dev, "%s: notifications disabled\n", 1870 nvdimm_name(nvdimm)); 1871 } 1872 1873 return devm_add_action_or_reset(acpi_desc->dev, shutdown_dimm_notify, 1874 acpi_desc); 1875 } 1876 1877 /* 1878 * These constants are private because there are no kernel consumers of 1879 * these commands. 1880 */ 1881 enum nfit_aux_cmds { 1882 NFIT_CMD_TRANSLATE_SPA = 5, 1883 NFIT_CMD_ARS_INJECT_SET = 7, 1884 NFIT_CMD_ARS_INJECT_CLEAR = 8, 1885 NFIT_CMD_ARS_INJECT_GET = 9, 1886 }; 1887 1888 static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc) 1889 { 1890 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 1891 const guid_t *guid = to_nfit_uuid(NFIT_DEV_BUS); 1892 struct acpi_device *adev; 1893 unsigned long dsm_mask; 1894 int i; 1895 1896 nd_desc->cmd_mask = acpi_desc->bus_cmd_force_en; 1897 nd_desc->bus_dsm_mask = acpi_desc->bus_nfit_cmd_force_en; 1898 adev = to_acpi_dev(acpi_desc); 1899 if (!adev) 1900 return; 1901 1902 for (i = ND_CMD_ARS_CAP; i <= ND_CMD_CLEAR_ERROR; i++) 1903 if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i)) 1904 set_bit(i, &nd_desc->cmd_mask); 1905 set_bit(ND_CMD_CALL, &nd_desc->cmd_mask); 1906 1907 dsm_mask = 1908 (1 << ND_CMD_ARS_CAP) | 1909 (1 << ND_CMD_ARS_START) | 1910 (1 << ND_CMD_ARS_STATUS) | 1911 (1 << ND_CMD_CLEAR_ERROR) | 1912 (1 << NFIT_CMD_TRANSLATE_SPA) | 1913 (1 << NFIT_CMD_ARS_INJECT_SET) | 1914 (1 << NFIT_CMD_ARS_INJECT_CLEAR) | 1915 (1 << NFIT_CMD_ARS_INJECT_GET); 1916 for_each_set_bit(i, &dsm_mask, BITS_PER_LONG) 1917 if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i)) 1918 set_bit(i, &nd_desc->bus_dsm_mask); 1919 } 1920 1921 static ssize_t range_index_show(struct device *dev, 1922 struct device_attribute *attr, char *buf) 1923 { 1924 struct nd_region *nd_region = to_nd_region(dev); 1925 struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region); 1926 1927 return sprintf(buf, "%d\n", nfit_spa->spa->range_index); 1928 } 1929 static DEVICE_ATTR_RO(range_index); 1930 1931 static ssize_t ecc_unit_size_show(struct device *dev, 1932 struct device_attribute *attr, char *buf) 1933 { 1934 struct nd_region *nd_region = to_nd_region(dev); 1935 struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region); 1936 1937 return sprintf(buf, "%d\n", nfit_spa->clear_err_unit); 1938 } 1939 static DEVICE_ATTR_RO(ecc_unit_size); 1940 1941 static struct attribute *acpi_nfit_region_attributes[] = { 1942 &dev_attr_range_index.attr, 1943 &dev_attr_ecc_unit_size.attr, 1944 NULL, 1945 }; 1946 1947 static const struct attribute_group acpi_nfit_region_attribute_group = { 1948 .name = "nfit", 1949 .attrs = acpi_nfit_region_attributes, 1950 }; 1951 1952 static const struct attribute_group *acpi_nfit_region_attribute_groups[] = { 1953 &nd_region_attribute_group, 1954 &nd_mapping_attribute_group, 1955 &nd_device_attribute_group, 1956 &nd_numa_attribute_group, 1957 &acpi_nfit_region_attribute_group, 1958 NULL, 1959 }; 1960 1961 /* enough info to uniquely specify an interleave set */ 1962 struct nfit_set_info { 1963 struct nfit_set_info_map { 1964 u64 region_offset; 1965 u32 serial_number; 1966 u32 pad; 1967 } mapping[0]; 1968 }; 1969 1970 struct nfit_set_info2 { 1971 struct nfit_set_info_map2 { 1972 u64 region_offset; 1973 u32 serial_number; 1974 u16 vendor_id; 1975 u16 manufacturing_date; 1976 u8 manufacturing_location; 1977 u8 reserved[31]; 1978 } mapping[0]; 1979 }; 1980 1981 static size_t sizeof_nfit_set_info(int num_mappings) 1982 { 1983 return sizeof(struct nfit_set_info) 1984 + num_mappings * sizeof(struct nfit_set_info_map); 1985 } 1986 1987 static size_t sizeof_nfit_set_info2(int num_mappings) 1988 { 1989 return sizeof(struct nfit_set_info2) 1990 + num_mappings * sizeof(struct nfit_set_info_map2); 1991 } 1992 1993 static int cmp_map_compat(const void *m0, const void *m1) 1994 { 1995 const struct nfit_set_info_map *map0 = m0; 1996 const struct nfit_set_info_map *map1 = m1; 1997 1998 return memcmp(&map0->region_offset, &map1->region_offset, 1999 sizeof(u64)); 2000 } 2001 2002 static int cmp_map(const void *m0, const void *m1) 2003 { 2004 const struct nfit_set_info_map *map0 = m0; 2005 const struct nfit_set_info_map *map1 = m1; 2006 2007 if (map0->region_offset < map1->region_offset) 2008 return -1; 2009 else if (map0->region_offset > map1->region_offset) 2010 return 1; 2011 return 0; 2012 } 2013 2014 static int cmp_map2(const void *m0, const void *m1) 2015 { 2016 const struct nfit_set_info_map2 *map0 = m0; 2017 const struct nfit_set_info_map2 *map1 = m1; 2018 2019 if (map0->region_offset < map1->region_offset) 2020 return -1; 2021 else if (map0->region_offset > map1->region_offset) 2022 return 1; 2023 return 0; 2024 } 2025 2026 /* Retrieve the nth entry referencing this spa */ 2027 static struct acpi_nfit_memory_map *memdev_from_spa( 2028 struct acpi_nfit_desc *acpi_desc, u16 range_index, int n) 2029 { 2030 struct nfit_memdev *nfit_memdev; 2031 2032 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) 2033 if (nfit_memdev->memdev->range_index == range_index) 2034 if (n-- == 0) 2035 return nfit_memdev->memdev; 2036 return NULL; 2037 } 2038 2039 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc, 2040 struct nd_region_desc *ndr_desc, 2041 struct acpi_nfit_system_address *spa) 2042 { 2043 struct device *dev = acpi_desc->dev; 2044 struct nd_interleave_set *nd_set; 2045 u16 nr = ndr_desc->num_mappings; 2046 struct nfit_set_info2 *info2; 2047 struct nfit_set_info *info; 2048 int i; 2049 2050 nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL); 2051 if (!nd_set) 2052 return -ENOMEM; 2053 ndr_desc->nd_set = nd_set; 2054 guid_copy(&nd_set->type_guid, (guid_t *) spa->range_guid); 2055 2056 info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL); 2057 if (!info) 2058 return -ENOMEM; 2059 2060 info2 = devm_kzalloc(dev, sizeof_nfit_set_info2(nr), GFP_KERNEL); 2061 if (!info2) 2062 return -ENOMEM; 2063 2064 for (i = 0; i < nr; i++) { 2065 struct nd_mapping_desc *mapping = &ndr_desc->mapping[i]; 2066 struct nfit_set_info_map *map = &info->mapping[i]; 2067 struct nfit_set_info_map2 *map2 = &info2->mapping[i]; 2068 struct nvdimm *nvdimm = mapping->nvdimm; 2069 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 2070 struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc, 2071 spa->range_index, i); 2072 struct acpi_nfit_control_region *dcr = nfit_mem->dcr; 2073 2074 if (!memdev || !nfit_mem->dcr) { 2075 dev_err(dev, "%s: failed to find DCR\n", __func__); 2076 return -ENODEV; 2077 } 2078 2079 map->region_offset = memdev->region_offset; 2080 map->serial_number = dcr->serial_number; 2081 2082 map2->region_offset = memdev->region_offset; 2083 map2->serial_number = dcr->serial_number; 2084 map2->vendor_id = dcr->vendor_id; 2085 map2->manufacturing_date = dcr->manufacturing_date; 2086 map2->manufacturing_location = dcr->manufacturing_location; 2087 } 2088 2089 /* v1.1 namespaces */ 2090 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map), 2091 cmp_map, NULL); 2092 nd_set->cookie1 = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0); 2093 2094 /* v1.2 namespaces */ 2095 sort(&info2->mapping[0], nr, sizeof(struct nfit_set_info_map2), 2096 cmp_map2, NULL); 2097 nd_set->cookie2 = nd_fletcher64(info2, sizeof_nfit_set_info2(nr), 0); 2098 2099 /* support v1.1 namespaces created with the wrong sort order */ 2100 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map), 2101 cmp_map_compat, NULL); 2102 nd_set->altcookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0); 2103 2104 /* record the result of the sort for the mapping position */ 2105 for (i = 0; i < nr; i++) { 2106 struct nfit_set_info_map2 *map2 = &info2->mapping[i]; 2107 int j; 2108 2109 for (j = 0; j < nr; j++) { 2110 struct nd_mapping_desc *mapping = &ndr_desc->mapping[j]; 2111 struct nvdimm *nvdimm = mapping->nvdimm; 2112 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 2113 struct acpi_nfit_control_region *dcr = nfit_mem->dcr; 2114 2115 if (map2->serial_number == dcr->serial_number && 2116 map2->vendor_id == dcr->vendor_id && 2117 map2->manufacturing_date == dcr->manufacturing_date && 2118 map2->manufacturing_location 2119 == dcr->manufacturing_location) { 2120 mapping->position = i; 2121 break; 2122 } 2123 } 2124 } 2125 2126 ndr_desc->nd_set = nd_set; 2127 devm_kfree(dev, info); 2128 devm_kfree(dev, info2); 2129 2130 return 0; 2131 } 2132 2133 static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio) 2134 { 2135 struct acpi_nfit_interleave *idt = mmio->idt; 2136 u32 sub_line_offset, line_index, line_offset; 2137 u64 line_no, table_skip_count, table_offset; 2138 2139 line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset); 2140 table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index); 2141 line_offset = idt->line_offset[line_index] 2142 * mmio->line_size; 2143 table_offset = table_skip_count * mmio->table_size; 2144 2145 return mmio->base_offset + line_offset + table_offset + sub_line_offset; 2146 } 2147 2148 static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw) 2149 { 2150 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; 2151 u64 offset = nfit_blk->stat_offset + mmio->size * bw; 2152 const u32 STATUS_MASK = 0x80000037; 2153 2154 if (mmio->num_lines) 2155 offset = to_interleave_offset(offset, mmio); 2156 2157 return readl(mmio->addr.base + offset) & STATUS_MASK; 2158 } 2159 2160 static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw, 2161 resource_size_t dpa, unsigned int len, unsigned int write) 2162 { 2163 u64 cmd, offset; 2164 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; 2165 2166 enum { 2167 BCW_OFFSET_MASK = (1ULL << 48)-1, 2168 BCW_LEN_SHIFT = 48, 2169 BCW_LEN_MASK = (1ULL << 8) - 1, 2170 BCW_CMD_SHIFT = 56, 2171 }; 2172 2173 cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK; 2174 len = len >> L1_CACHE_SHIFT; 2175 cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT; 2176 cmd |= ((u64) write) << BCW_CMD_SHIFT; 2177 2178 offset = nfit_blk->cmd_offset + mmio->size * bw; 2179 if (mmio->num_lines) 2180 offset = to_interleave_offset(offset, mmio); 2181 2182 writeq(cmd, mmio->addr.base + offset); 2183 nvdimm_flush(nfit_blk->nd_region); 2184 2185 if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH) 2186 readq(mmio->addr.base + offset); 2187 } 2188 2189 static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk, 2190 resource_size_t dpa, void *iobuf, size_t len, int rw, 2191 unsigned int lane) 2192 { 2193 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW]; 2194 unsigned int copied = 0; 2195 u64 base_offset; 2196 int rc; 2197 2198 base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES 2199 + lane * mmio->size; 2200 write_blk_ctl(nfit_blk, lane, dpa, len, rw); 2201 while (len) { 2202 unsigned int c; 2203 u64 offset; 2204 2205 if (mmio->num_lines) { 2206 u32 line_offset; 2207 2208 offset = to_interleave_offset(base_offset + copied, 2209 mmio); 2210 div_u64_rem(offset, mmio->line_size, &line_offset); 2211 c = min_t(size_t, len, mmio->line_size - line_offset); 2212 } else { 2213 offset = base_offset + nfit_blk->bdw_offset; 2214 c = len; 2215 } 2216 2217 if (rw) 2218 memcpy_flushcache(mmio->addr.aperture + offset, iobuf + copied, c); 2219 else { 2220 if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH) 2221 arch_invalidate_pmem((void __force *) 2222 mmio->addr.aperture + offset, c); 2223 2224 memcpy(iobuf + copied, mmio->addr.aperture + offset, c); 2225 } 2226 2227 copied += c; 2228 len -= c; 2229 } 2230 2231 if (rw) 2232 nvdimm_flush(nfit_blk->nd_region); 2233 2234 rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0; 2235 return rc; 2236 } 2237 2238 static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr, 2239 resource_size_t dpa, void *iobuf, u64 len, int rw) 2240 { 2241 struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr); 2242 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW]; 2243 struct nd_region *nd_region = nfit_blk->nd_region; 2244 unsigned int lane, copied = 0; 2245 int rc = 0; 2246 2247 lane = nd_region_acquire_lane(nd_region); 2248 while (len) { 2249 u64 c = min(len, mmio->size); 2250 2251 rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied, 2252 iobuf + copied, c, rw, lane); 2253 if (rc) 2254 break; 2255 2256 copied += c; 2257 len -= c; 2258 } 2259 nd_region_release_lane(nd_region, lane); 2260 2261 return rc; 2262 } 2263 2264 static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio, 2265 struct acpi_nfit_interleave *idt, u16 interleave_ways) 2266 { 2267 if (idt) { 2268 mmio->num_lines = idt->line_count; 2269 mmio->line_size = idt->line_size; 2270 if (interleave_ways == 0) 2271 return -ENXIO; 2272 mmio->table_size = mmio->num_lines * interleave_ways 2273 * mmio->line_size; 2274 } 2275 2276 return 0; 2277 } 2278 2279 static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc, 2280 struct nvdimm *nvdimm, struct nfit_blk *nfit_blk) 2281 { 2282 struct nd_cmd_dimm_flags flags; 2283 int rc; 2284 2285 memset(&flags, 0, sizeof(flags)); 2286 rc = nd_desc->ndctl(nd_desc, nvdimm, ND_CMD_DIMM_FLAGS, &flags, 2287 sizeof(flags), NULL); 2288 2289 if (rc >= 0 && flags.status == 0) 2290 nfit_blk->dimm_flags = flags.flags; 2291 else if (rc == -ENOTTY) { 2292 /* fall back to a conservative default */ 2293 nfit_blk->dimm_flags = NFIT_BLK_DCR_LATCH | NFIT_BLK_READ_FLUSH; 2294 rc = 0; 2295 } else 2296 rc = -ENXIO; 2297 2298 return rc; 2299 } 2300 2301 static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, 2302 struct device *dev) 2303 { 2304 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 2305 struct nd_blk_region *ndbr = to_nd_blk_region(dev); 2306 struct nfit_blk_mmio *mmio; 2307 struct nfit_blk *nfit_blk; 2308 struct nfit_mem *nfit_mem; 2309 struct nvdimm *nvdimm; 2310 int rc; 2311 2312 nvdimm = nd_blk_region_to_dimm(ndbr); 2313 nfit_mem = nvdimm_provider_data(nvdimm); 2314 if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) { 2315 dev_dbg(dev, "%s: missing%s%s%s\n", __func__, 2316 nfit_mem ? "" : " nfit_mem", 2317 (nfit_mem && nfit_mem->dcr) ? "" : " dcr", 2318 (nfit_mem && nfit_mem->bdw) ? "" : " bdw"); 2319 return -ENXIO; 2320 } 2321 2322 nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL); 2323 if (!nfit_blk) 2324 return -ENOMEM; 2325 nd_blk_region_set_provider_data(ndbr, nfit_blk); 2326 nfit_blk->nd_region = to_nd_region(dev); 2327 2328 /* map block aperture memory */ 2329 nfit_blk->bdw_offset = nfit_mem->bdw->offset; 2330 mmio = &nfit_blk->mmio[BDW]; 2331 mmio->addr.base = devm_nvdimm_memremap(dev, nfit_mem->spa_bdw->address, 2332 nfit_mem->spa_bdw->length, nd_blk_memremap_flags(ndbr)); 2333 if (!mmio->addr.base) { 2334 dev_dbg(dev, "%s: %s failed to map bdw\n", __func__, 2335 nvdimm_name(nvdimm)); 2336 return -ENOMEM; 2337 } 2338 mmio->size = nfit_mem->bdw->size; 2339 mmio->base_offset = nfit_mem->memdev_bdw->region_offset; 2340 mmio->idt = nfit_mem->idt_bdw; 2341 mmio->spa = nfit_mem->spa_bdw; 2342 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw, 2343 nfit_mem->memdev_bdw->interleave_ways); 2344 if (rc) { 2345 dev_dbg(dev, "%s: %s failed to init bdw interleave\n", 2346 __func__, nvdimm_name(nvdimm)); 2347 return rc; 2348 } 2349 2350 /* map block control memory */ 2351 nfit_blk->cmd_offset = nfit_mem->dcr->command_offset; 2352 nfit_blk->stat_offset = nfit_mem->dcr->status_offset; 2353 mmio = &nfit_blk->mmio[DCR]; 2354 mmio->addr.base = devm_nvdimm_ioremap(dev, nfit_mem->spa_dcr->address, 2355 nfit_mem->spa_dcr->length); 2356 if (!mmio->addr.base) { 2357 dev_dbg(dev, "%s: %s failed to map dcr\n", __func__, 2358 nvdimm_name(nvdimm)); 2359 return -ENOMEM; 2360 } 2361 mmio->size = nfit_mem->dcr->window_size; 2362 mmio->base_offset = nfit_mem->memdev_dcr->region_offset; 2363 mmio->idt = nfit_mem->idt_dcr; 2364 mmio->spa = nfit_mem->spa_dcr; 2365 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr, 2366 nfit_mem->memdev_dcr->interleave_ways); 2367 if (rc) { 2368 dev_dbg(dev, "%s: %s failed to init dcr interleave\n", 2369 __func__, nvdimm_name(nvdimm)); 2370 return rc; 2371 } 2372 2373 rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk); 2374 if (rc < 0) { 2375 dev_dbg(dev, "%s: %s failed get DIMM flags\n", 2376 __func__, nvdimm_name(nvdimm)); 2377 return rc; 2378 } 2379 2380 if (nvdimm_has_flush(nfit_blk->nd_region) < 0) 2381 dev_warn(dev, "unable to guarantee persistence of writes\n"); 2382 2383 if (mmio->line_size == 0) 2384 return 0; 2385 2386 if ((u32) nfit_blk->cmd_offset % mmio->line_size 2387 + 8 > mmio->line_size) { 2388 dev_dbg(dev, "cmd_offset crosses interleave boundary\n"); 2389 return -ENXIO; 2390 } else if ((u32) nfit_blk->stat_offset % mmio->line_size 2391 + 8 > mmio->line_size) { 2392 dev_dbg(dev, "stat_offset crosses interleave boundary\n"); 2393 return -ENXIO; 2394 } 2395 2396 return 0; 2397 } 2398 2399 static int ars_get_cap(struct acpi_nfit_desc *acpi_desc, 2400 struct nd_cmd_ars_cap *cmd, struct nfit_spa *nfit_spa) 2401 { 2402 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2403 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2404 int cmd_rc, rc; 2405 2406 cmd->address = spa->address; 2407 cmd->length = spa->length; 2408 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, cmd, 2409 sizeof(*cmd), &cmd_rc); 2410 if (rc < 0) 2411 return rc; 2412 return cmd_rc; 2413 } 2414 2415 static int ars_start(struct acpi_nfit_desc *acpi_desc, struct nfit_spa *nfit_spa) 2416 { 2417 int rc; 2418 int cmd_rc; 2419 struct nd_cmd_ars_start ars_start; 2420 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2421 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2422 2423 memset(&ars_start, 0, sizeof(ars_start)); 2424 ars_start.address = spa->address; 2425 ars_start.length = spa->length; 2426 ars_start.flags = acpi_desc->ars_start_flags; 2427 if (nfit_spa_type(spa) == NFIT_SPA_PM) 2428 ars_start.type = ND_ARS_PERSISTENT; 2429 else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE) 2430 ars_start.type = ND_ARS_VOLATILE; 2431 else 2432 return -ENOTTY; 2433 2434 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start, 2435 sizeof(ars_start), &cmd_rc); 2436 2437 if (rc < 0) 2438 return rc; 2439 return cmd_rc; 2440 } 2441 2442 static int ars_continue(struct acpi_nfit_desc *acpi_desc) 2443 { 2444 int rc, cmd_rc; 2445 struct nd_cmd_ars_start ars_start; 2446 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2447 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; 2448 2449 memset(&ars_start, 0, sizeof(ars_start)); 2450 ars_start.address = ars_status->restart_address; 2451 ars_start.length = ars_status->restart_length; 2452 ars_start.type = ars_status->type; 2453 ars_start.flags = acpi_desc->ars_start_flags; 2454 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start, 2455 sizeof(ars_start), &cmd_rc); 2456 if (rc < 0) 2457 return rc; 2458 return cmd_rc; 2459 } 2460 2461 static int ars_get_status(struct acpi_nfit_desc *acpi_desc) 2462 { 2463 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2464 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; 2465 int rc, cmd_rc; 2466 2467 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_STATUS, ars_status, 2468 acpi_desc->ars_status_size, &cmd_rc); 2469 if (rc < 0) 2470 return rc; 2471 return cmd_rc; 2472 } 2473 2474 static int ars_status_process_records(struct acpi_nfit_desc *acpi_desc, 2475 struct nd_cmd_ars_status *ars_status) 2476 { 2477 struct nvdimm_bus *nvdimm_bus = acpi_desc->nvdimm_bus; 2478 int rc; 2479 u32 i; 2480 2481 /* 2482 * First record starts at 44 byte offset from the start of the 2483 * payload. 2484 */ 2485 if (ars_status->out_length < 44) 2486 return 0; 2487 for (i = 0; i < ars_status->num_records; i++) { 2488 /* only process full records */ 2489 if (ars_status->out_length 2490 < 44 + sizeof(struct nd_ars_record) * (i + 1)) 2491 break; 2492 rc = nvdimm_bus_add_badrange(nvdimm_bus, 2493 ars_status->records[i].err_address, 2494 ars_status->records[i].length); 2495 if (rc) 2496 return rc; 2497 } 2498 if (i < ars_status->num_records) 2499 dev_warn(acpi_desc->dev, "detected truncated ars results\n"); 2500 2501 return 0; 2502 } 2503 2504 static void acpi_nfit_remove_resource(void *data) 2505 { 2506 struct resource *res = data; 2507 2508 remove_resource(res); 2509 } 2510 2511 static int acpi_nfit_insert_resource(struct acpi_nfit_desc *acpi_desc, 2512 struct nd_region_desc *ndr_desc) 2513 { 2514 struct resource *res, *nd_res = ndr_desc->res; 2515 int is_pmem, ret; 2516 2517 /* No operation if the region is already registered as PMEM */ 2518 is_pmem = region_intersects(nd_res->start, resource_size(nd_res), 2519 IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY); 2520 if (is_pmem == REGION_INTERSECTS) 2521 return 0; 2522 2523 res = devm_kzalloc(acpi_desc->dev, sizeof(*res), GFP_KERNEL); 2524 if (!res) 2525 return -ENOMEM; 2526 2527 res->name = "Persistent Memory"; 2528 res->start = nd_res->start; 2529 res->end = nd_res->end; 2530 res->flags = IORESOURCE_MEM; 2531 res->desc = IORES_DESC_PERSISTENT_MEMORY; 2532 2533 ret = insert_resource(&iomem_resource, res); 2534 if (ret) 2535 return ret; 2536 2537 ret = devm_add_action_or_reset(acpi_desc->dev, 2538 acpi_nfit_remove_resource, 2539 res); 2540 if (ret) 2541 return ret; 2542 2543 return 0; 2544 } 2545 2546 static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc, 2547 struct nd_mapping_desc *mapping, struct nd_region_desc *ndr_desc, 2548 struct acpi_nfit_memory_map *memdev, 2549 struct nfit_spa *nfit_spa) 2550 { 2551 struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, 2552 memdev->device_handle); 2553 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2554 struct nd_blk_region_desc *ndbr_desc; 2555 struct nfit_mem *nfit_mem; 2556 int blk_valid = 0, rc; 2557 2558 if (!nvdimm) { 2559 dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n", 2560 spa->range_index, memdev->device_handle); 2561 return -ENODEV; 2562 } 2563 2564 mapping->nvdimm = nvdimm; 2565 switch (nfit_spa_type(spa)) { 2566 case NFIT_SPA_PM: 2567 case NFIT_SPA_VOLATILE: 2568 mapping->start = memdev->address; 2569 mapping->size = memdev->region_size; 2570 break; 2571 case NFIT_SPA_DCR: 2572 nfit_mem = nvdimm_provider_data(nvdimm); 2573 if (!nfit_mem || !nfit_mem->bdw) { 2574 dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n", 2575 spa->range_index, nvdimm_name(nvdimm)); 2576 } else { 2577 mapping->size = nfit_mem->bdw->capacity; 2578 mapping->start = nfit_mem->bdw->start_address; 2579 ndr_desc->num_lanes = nfit_mem->bdw->windows; 2580 blk_valid = 1; 2581 } 2582 2583 ndr_desc->mapping = mapping; 2584 ndr_desc->num_mappings = blk_valid; 2585 ndbr_desc = to_blk_region_desc(ndr_desc); 2586 ndbr_desc->enable = acpi_nfit_blk_region_enable; 2587 ndbr_desc->do_io = acpi_desc->blk_do_io; 2588 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa); 2589 if (rc) 2590 return rc; 2591 nfit_spa->nd_region = nvdimm_blk_region_create(acpi_desc->nvdimm_bus, 2592 ndr_desc); 2593 if (!nfit_spa->nd_region) 2594 return -ENOMEM; 2595 break; 2596 } 2597 2598 return 0; 2599 } 2600 2601 static bool nfit_spa_is_virtual(struct acpi_nfit_system_address *spa) 2602 { 2603 return (nfit_spa_type(spa) == NFIT_SPA_VDISK || 2604 nfit_spa_type(spa) == NFIT_SPA_VCD || 2605 nfit_spa_type(spa) == NFIT_SPA_PDISK || 2606 nfit_spa_type(spa) == NFIT_SPA_PCD); 2607 } 2608 2609 static bool nfit_spa_is_volatile(struct acpi_nfit_system_address *spa) 2610 { 2611 return (nfit_spa_type(spa) == NFIT_SPA_VDISK || 2612 nfit_spa_type(spa) == NFIT_SPA_VCD || 2613 nfit_spa_type(spa) == NFIT_SPA_VOLATILE); 2614 } 2615 2616 static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc, 2617 struct nfit_spa *nfit_spa) 2618 { 2619 static struct nd_mapping_desc mappings[ND_MAX_MAPPINGS]; 2620 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2621 struct nd_blk_region_desc ndbr_desc; 2622 struct nd_region_desc *ndr_desc; 2623 struct nfit_memdev *nfit_memdev; 2624 struct nvdimm_bus *nvdimm_bus; 2625 struct resource res; 2626 int count = 0, rc; 2627 2628 if (nfit_spa->nd_region) 2629 return 0; 2630 2631 if (spa->range_index == 0 && !nfit_spa_is_virtual(spa)) { 2632 dev_dbg(acpi_desc->dev, "%s: detected invalid spa index\n", 2633 __func__); 2634 return 0; 2635 } 2636 2637 memset(&res, 0, sizeof(res)); 2638 memset(&mappings, 0, sizeof(mappings)); 2639 memset(&ndbr_desc, 0, sizeof(ndbr_desc)); 2640 res.start = spa->address; 2641 res.end = res.start + spa->length - 1; 2642 ndr_desc = &ndbr_desc.ndr_desc; 2643 ndr_desc->res = &res; 2644 ndr_desc->provider_data = nfit_spa; 2645 ndr_desc->attr_groups = acpi_nfit_region_attribute_groups; 2646 if (spa->flags & ACPI_NFIT_PROXIMITY_VALID) 2647 ndr_desc->numa_node = acpi_map_pxm_to_online_node( 2648 spa->proximity_domain); 2649 else 2650 ndr_desc->numa_node = NUMA_NO_NODE; 2651 2652 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 2653 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev; 2654 struct nd_mapping_desc *mapping; 2655 2656 if (memdev->range_index != spa->range_index) 2657 continue; 2658 if (count >= ND_MAX_MAPPINGS) { 2659 dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n", 2660 spa->range_index, ND_MAX_MAPPINGS); 2661 return -ENXIO; 2662 } 2663 mapping = &mappings[count++]; 2664 rc = acpi_nfit_init_mapping(acpi_desc, mapping, ndr_desc, 2665 memdev, nfit_spa); 2666 if (rc) 2667 goto out; 2668 } 2669 2670 ndr_desc->mapping = mappings; 2671 ndr_desc->num_mappings = count; 2672 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa); 2673 if (rc) 2674 goto out; 2675 2676 nvdimm_bus = acpi_desc->nvdimm_bus; 2677 if (nfit_spa_type(spa) == NFIT_SPA_PM) { 2678 rc = acpi_nfit_insert_resource(acpi_desc, ndr_desc); 2679 if (rc) { 2680 dev_warn(acpi_desc->dev, 2681 "failed to insert pmem resource to iomem: %d\n", 2682 rc); 2683 goto out; 2684 } 2685 2686 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus, 2687 ndr_desc); 2688 if (!nfit_spa->nd_region) 2689 rc = -ENOMEM; 2690 } else if (nfit_spa_is_volatile(spa)) { 2691 nfit_spa->nd_region = nvdimm_volatile_region_create(nvdimm_bus, 2692 ndr_desc); 2693 if (!nfit_spa->nd_region) 2694 rc = -ENOMEM; 2695 } else if (nfit_spa_is_virtual(spa)) { 2696 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus, 2697 ndr_desc); 2698 if (!nfit_spa->nd_region) 2699 rc = -ENOMEM; 2700 } 2701 2702 out: 2703 if (rc) 2704 dev_err(acpi_desc->dev, "failed to register spa range %d\n", 2705 nfit_spa->spa->range_index); 2706 return rc; 2707 } 2708 2709 static int ars_status_alloc(struct acpi_nfit_desc *acpi_desc, 2710 u32 max_ars) 2711 { 2712 struct device *dev = acpi_desc->dev; 2713 struct nd_cmd_ars_status *ars_status; 2714 2715 if (acpi_desc->ars_status && acpi_desc->ars_status_size >= max_ars) { 2716 memset(acpi_desc->ars_status, 0, acpi_desc->ars_status_size); 2717 return 0; 2718 } 2719 2720 if (acpi_desc->ars_status) 2721 devm_kfree(dev, acpi_desc->ars_status); 2722 acpi_desc->ars_status = NULL; 2723 ars_status = devm_kzalloc(dev, max_ars, GFP_KERNEL); 2724 if (!ars_status) 2725 return -ENOMEM; 2726 acpi_desc->ars_status = ars_status; 2727 acpi_desc->ars_status_size = max_ars; 2728 return 0; 2729 } 2730 2731 static int acpi_nfit_query_poison(struct acpi_nfit_desc *acpi_desc, 2732 struct nfit_spa *nfit_spa) 2733 { 2734 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2735 int rc; 2736 2737 if (!nfit_spa->max_ars) { 2738 struct nd_cmd_ars_cap ars_cap; 2739 2740 memset(&ars_cap, 0, sizeof(ars_cap)); 2741 rc = ars_get_cap(acpi_desc, &ars_cap, nfit_spa); 2742 if (rc < 0) 2743 return rc; 2744 nfit_spa->max_ars = ars_cap.max_ars_out; 2745 nfit_spa->clear_err_unit = ars_cap.clear_err_unit; 2746 /* check that the supported scrub types match the spa type */ 2747 if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE && 2748 ((ars_cap.status >> 16) & ND_ARS_VOLATILE) == 0) 2749 return -ENOTTY; 2750 else if (nfit_spa_type(spa) == NFIT_SPA_PM && 2751 ((ars_cap.status >> 16) & ND_ARS_PERSISTENT) == 0) 2752 return -ENOTTY; 2753 } 2754 2755 if (ars_status_alloc(acpi_desc, nfit_spa->max_ars)) 2756 return -ENOMEM; 2757 2758 rc = ars_get_status(acpi_desc); 2759 if (rc < 0 && rc != -ENOSPC) 2760 return rc; 2761 2762 if (ars_status_process_records(acpi_desc, acpi_desc->ars_status)) 2763 return -ENOMEM; 2764 2765 return 0; 2766 } 2767 2768 static void acpi_nfit_async_scrub(struct acpi_nfit_desc *acpi_desc, 2769 struct nfit_spa *nfit_spa) 2770 { 2771 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2772 unsigned int overflow_retry = scrub_overflow_abort; 2773 u64 init_ars_start = 0, init_ars_len = 0; 2774 struct device *dev = acpi_desc->dev; 2775 unsigned int tmo = scrub_timeout; 2776 int rc; 2777 2778 if (!nfit_spa->ars_required || !nfit_spa->nd_region) 2779 return; 2780 2781 rc = ars_start(acpi_desc, nfit_spa); 2782 /* 2783 * If we timed out the initial scan we'll still be busy here, 2784 * and will wait another timeout before giving up permanently. 2785 */ 2786 if (rc < 0 && rc != -EBUSY) 2787 return; 2788 2789 do { 2790 u64 ars_start, ars_len; 2791 2792 if (acpi_desc->cancel) 2793 break; 2794 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa); 2795 if (rc == -ENOTTY) 2796 break; 2797 if (rc == -EBUSY && !tmo) { 2798 dev_warn(dev, "range %d ars timeout, aborting\n", 2799 spa->range_index); 2800 break; 2801 } 2802 2803 if (rc == -EBUSY) { 2804 /* 2805 * Note, entries may be appended to the list 2806 * while the lock is dropped, but the workqueue 2807 * being active prevents entries being deleted / 2808 * freed. 2809 */ 2810 mutex_unlock(&acpi_desc->init_mutex); 2811 ssleep(1); 2812 tmo--; 2813 mutex_lock(&acpi_desc->init_mutex); 2814 continue; 2815 } 2816 2817 /* we got some results, but there are more pending... */ 2818 if (rc == -ENOSPC && overflow_retry--) { 2819 if (!init_ars_len) { 2820 init_ars_len = acpi_desc->ars_status->length; 2821 init_ars_start = acpi_desc->ars_status->address; 2822 } 2823 rc = ars_continue(acpi_desc); 2824 } 2825 2826 if (rc < 0) { 2827 dev_warn(dev, "range %d ars continuation failed\n", 2828 spa->range_index); 2829 break; 2830 } 2831 2832 if (init_ars_len) { 2833 ars_start = init_ars_start; 2834 ars_len = init_ars_len; 2835 } else { 2836 ars_start = acpi_desc->ars_status->address; 2837 ars_len = acpi_desc->ars_status->length; 2838 } 2839 dev_dbg(dev, "spa range: %d ars from %#llx + %#llx complete\n", 2840 spa->range_index, ars_start, ars_len); 2841 /* notify the region about new poison entries */ 2842 nvdimm_region_notify(nfit_spa->nd_region, 2843 NVDIMM_REVALIDATE_POISON); 2844 break; 2845 } while (1); 2846 } 2847 2848 static void acpi_nfit_scrub(struct work_struct *work) 2849 { 2850 struct device *dev; 2851 u64 init_scrub_length = 0; 2852 struct nfit_spa *nfit_spa; 2853 u64 init_scrub_address = 0; 2854 bool init_ars_done = false; 2855 struct acpi_nfit_desc *acpi_desc; 2856 unsigned int tmo = scrub_timeout; 2857 unsigned int overflow_retry = scrub_overflow_abort; 2858 2859 acpi_desc = container_of(work, typeof(*acpi_desc), work); 2860 dev = acpi_desc->dev; 2861 2862 /* 2863 * We scrub in 2 phases. The first phase waits for any platform 2864 * firmware initiated scrubs to complete and then we go search for the 2865 * affected spa regions to mark them scanned. In the second phase we 2866 * initiate a directed scrub for every range that was not scrubbed in 2867 * phase 1. If we're called for a 'rescan', we harmlessly pass through 2868 * the first phase, but really only care about running phase 2, where 2869 * regions can be notified of new poison. 2870 */ 2871 2872 /* process platform firmware initiated scrubs */ 2873 retry: 2874 mutex_lock(&acpi_desc->init_mutex); 2875 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 2876 struct nd_cmd_ars_status *ars_status; 2877 struct acpi_nfit_system_address *spa; 2878 u64 ars_start, ars_len; 2879 int rc; 2880 2881 if (acpi_desc->cancel) 2882 break; 2883 2884 if (nfit_spa->nd_region) 2885 continue; 2886 2887 if (init_ars_done) { 2888 /* 2889 * No need to re-query, we're now just 2890 * reconciling all the ranges covered by the 2891 * initial scrub 2892 */ 2893 rc = 0; 2894 } else 2895 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa); 2896 2897 if (rc == -ENOTTY) { 2898 /* no ars capability, just register spa and move on */ 2899 acpi_nfit_register_region(acpi_desc, nfit_spa); 2900 continue; 2901 } 2902 2903 if (rc == -EBUSY && !tmo) { 2904 /* fallthrough to directed scrub in phase 2 */ 2905 dev_warn(dev, "timeout awaiting ars results, continuing...\n"); 2906 break; 2907 } else if (rc == -EBUSY) { 2908 mutex_unlock(&acpi_desc->init_mutex); 2909 ssleep(1); 2910 tmo--; 2911 goto retry; 2912 } 2913 2914 /* we got some results, but there are more pending... */ 2915 if (rc == -ENOSPC && overflow_retry--) { 2916 ars_status = acpi_desc->ars_status; 2917 /* 2918 * Record the original scrub range, so that we 2919 * can recall all the ranges impacted by the 2920 * initial scrub. 2921 */ 2922 if (!init_scrub_length) { 2923 init_scrub_length = ars_status->length; 2924 init_scrub_address = ars_status->address; 2925 } 2926 rc = ars_continue(acpi_desc); 2927 if (rc == 0) { 2928 mutex_unlock(&acpi_desc->init_mutex); 2929 goto retry; 2930 } 2931 } 2932 2933 if (rc < 0) { 2934 /* 2935 * Initial scrub failed, we'll give it one more 2936 * try below... 2937 */ 2938 break; 2939 } 2940 2941 /* We got some final results, record completed ranges */ 2942 ars_status = acpi_desc->ars_status; 2943 if (init_scrub_length) { 2944 ars_start = init_scrub_address; 2945 ars_len = ars_start + init_scrub_length; 2946 } else { 2947 ars_start = ars_status->address; 2948 ars_len = ars_status->length; 2949 } 2950 spa = nfit_spa->spa; 2951 2952 if (!init_ars_done) { 2953 init_ars_done = true; 2954 dev_dbg(dev, "init scrub %#llx + %#llx complete\n", 2955 ars_start, ars_len); 2956 } 2957 if (ars_start <= spa->address && ars_start + ars_len 2958 >= spa->address + spa->length) 2959 acpi_nfit_register_region(acpi_desc, nfit_spa); 2960 } 2961 2962 /* 2963 * For all the ranges not covered by an initial scrub we still 2964 * want to see if there are errors, but it's ok to discover them 2965 * asynchronously. 2966 */ 2967 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 2968 /* 2969 * Flag all the ranges that still need scrubbing, but 2970 * register them now to make data available. 2971 */ 2972 if (!nfit_spa->nd_region) { 2973 nfit_spa->ars_required = 1; 2974 acpi_nfit_register_region(acpi_desc, nfit_spa); 2975 } 2976 } 2977 acpi_desc->init_complete = 1; 2978 2979 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) 2980 acpi_nfit_async_scrub(acpi_desc, nfit_spa); 2981 acpi_desc->scrub_count++; 2982 acpi_desc->ars_start_flags = 0; 2983 if (acpi_desc->scrub_count_state) 2984 sysfs_notify_dirent(acpi_desc->scrub_count_state); 2985 mutex_unlock(&acpi_desc->init_mutex); 2986 } 2987 2988 static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc) 2989 { 2990 struct nfit_spa *nfit_spa; 2991 int rc; 2992 2993 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) 2994 if (nfit_spa_type(nfit_spa->spa) == NFIT_SPA_DCR) { 2995 /* BLK regions don't need to wait for ars results */ 2996 rc = acpi_nfit_register_region(acpi_desc, nfit_spa); 2997 if (rc) 2998 return rc; 2999 } 3000 3001 acpi_desc->ars_start_flags = 0; 3002 if (!acpi_desc->cancel) 3003 queue_work(nfit_wq, &acpi_desc->work); 3004 return 0; 3005 } 3006 3007 static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc, 3008 struct nfit_table_prev *prev) 3009 { 3010 struct device *dev = acpi_desc->dev; 3011 3012 if (!list_empty(&prev->spas) || 3013 !list_empty(&prev->memdevs) || 3014 !list_empty(&prev->dcrs) || 3015 !list_empty(&prev->bdws) || 3016 !list_empty(&prev->idts) || 3017 !list_empty(&prev->flushes)) { 3018 dev_err(dev, "new nfit deletes entries (unsupported)\n"); 3019 return -ENXIO; 3020 } 3021 return 0; 3022 } 3023 3024 static int acpi_nfit_desc_init_scrub_attr(struct acpi_nfit_desc *acpi_desc) 3025 { 3026 struct device *dev = acpi_desc->dev; 3027 struct kernfs_node *nfit; 3028 struct device *bus_dev; 3029 3030 if (!ars_supported(acpi_desc->nvdimm_bus)) 3031 return 0; 3032 3033 bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); 3034 nfit = sysfs_get_dirent(bus_dev->kobj.sd, "nfit"); 3035 if (!nfit) { 3036 dev_err(dev, "sysfs_get_dirent 'nfit' failed\n"); 3037 return -ENODEV; 3038 } 3039 acpi_desc->scrub_count_state = sysfs_get_dirent(nfit, "scrub"); 3040 sysfs_put(nfit); 3041 if (!acpi_desc->scrub_count_state) { 3042 dev_err(dev, "sysfs_get_dirent 'scrub' failed\n"); 3043 return -ENODEV; 3044 } 3045 3046 return 0; 3047 } 3048 3049 static void acpi_nfit_unregister(void *data) 3050 { 3051 struct acpi_nfit_desc *acpi_desc = data; 3052 3053 nvdimm_bus_unregister(acpi_desc->nvdimm_bus); 3054 } 3055 3056 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz) 3057 { 3058 struct device *dev = acpi_desc->dev; 3059 struct nfit_table_prev prev; 3060 const void *end; 3061 int rc; 3062 3063 if (!acpi_desc->nvdimm_bus) { 3064 acpi_nfit_init_dsms(acpi_desc); 3065 3066 acpi_desc->nvdimm_bus = nvdimm_bus_register(dev, 3067 &acpi_desc->nd_desc); 3068 if (!acpi_desc->nvdimm_bus) 3069 return -ENOMEM; 3070 3071 rc = devm_add_action_or_reset(dev, acpi_nfit_unregister, 3072 acpi_desc); 3073 if (rc) 3074 return rc; 3075 3076 rc = acpi_nfit_desc_init_scrub_attr(acpi_desc); 3077 if (rc) 3078 return rc; 3079 3080 /* register this acpi_desc for mce notifications */ 3081 mutex_lock(&acpi_desc_lock); 3082 list_add_tail(&acpi_desc->list, &acpi_descs); 3083 mutex_unlock(&acpi_desc_lock); 3084 } 3085 3086 mutex_lock(&acpi_desc->init_mutex); 3087 3088 INIT_LIST_HEAD(&prev.spas); 3089 INIT_LIST_HEAD(&prev.memdevs); 3090 INIT_LIST_HEAD(&prev.dcrs); 3091 INIT_LIST_HEAD(&prev.bdws); 3092 INIT_LIST_HEAD(&prev.idts); 3093 INIT_LIST_HEAD(&prev.flushes); 3094 3095 list_cut_position(&prev.spas, &acpi_desc->spas, 3096 acpi_desc->spas.prev); 3097 list_cut_position(&prev.memdevs, &acpi_desc->memdevs, 3098 acpi_desc->memdevs.prev); 3099 list_cut_position(&prev.dcrs, &acpi_desc->dcrs, 3100 acpi_desc->dcrs.prev); 3101 list_cut_position(&prev.bdws, &acpi_desc->bdws, 3102 acpi_desc->bdws.prev); 3103 list_cut_position(&prev.idts, &acpi_desc->idts, 3104 acpi_desc->idts.prev); 3105 list_cut_position(&prev.flushes, &acpi_desc->flushes, 3106 acpi_desc->flushes.prev); 3107 3108 end = data + sz; 3109 while (!IS_ERR_OR_NULL(data)) 3110 data = add_table(acpi_desc, &prev, data, end); 3111 3112 if (IS_ERR(data)) { 3113 dev_dbg(dev, "%s: nfit table parsing error: %ld\n", __func__, 3114 PTR_ERR(data)); 3115 rc = PTR_ERR(data); 3116 goto out_unlock; 3117 } 3118 3119 rc = acpi_nfit_check_deletions(acpi_desc, &prev); 3120 if (rc) 3121 goto out_unlock; 3122 3123 rc = nfit_mem_init(acpi_desc); 3124 if (rc) 3125 goto out_unlock; 3126 3127 rc = acpi_nfit_register_dimms(acpi_desc); 3128 if (rc) 3129 goto out_unlock; 3130 3131 rc = acpi_nfit_register_regions(acpi_desc); 3132 3133 out_unlock: 3134 mutex_unlock(&acpi_desc->init_mutex); 3135 return rc; 3136 } 3137 EXPORT_SYMBOL_GPL(acpi_nfit_init); 3138 3139 struct acpi_nfit_flush_work { 3140 struct work_struct work; 3141 struct completion cmp; 3142 }; 3143 3144 static void flush_probe(struct work_struct *work) 3145 { 3146 struct acpi_nfit_flush_work *flush; 3147 3148 flush = container_of(work, typeof(*flush), work); 3149 complete(&flush->cmp); 3150 } 3151 3152 static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc) 3153 { 3154 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 3155 struct device *dev = acpi_desc->dev; 3156 struct acpi_nfit_flush_work flush; 3157 int rc; 3158 3159 /* bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */ 3160 device_lock(dev); 3161 device_unlock(dev); 3162 3163 /* bounce the init_mutex to make init_complete valid */ 3164 mutex_lock(&acpi_desc->init_mutex); 3165 if (acpi_desc->cancel || acpi_desc->init_complete) { 3166 mutex_unlock(&acpi_desc->init_mutex); 3167 return 0; 3168 } 3169 3170 /* 3171 * Scrub work could take 10s of seconds, userspace may give up so we 3172 * need to be interruptible while waiting. 3173 */ 3174 INIT_WORK_ONSTACK(&flush.work, flush_probe); 3175 init_completion(&flush.cmp); 3176 queue_work(nfit_wq, &flush.work); 3177 mutex_unlock(&acpi_desc->init_mutex); 3178 3179 rc = wait_for_completion_interruptible(&flush.cmp); 3180 cancel_work_sync(&flush.work); 3181 return rc; 3182 } 3183 3184 static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc, 3185 struct nvdimm *nvdimm, unsigned int cmd) 3186 { 3187 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 3188 3189 if (nvdimm) 3190 return 0; 3191 if (cmd != ND_CMD_ARS_START) 3192 return 0; 3193 3194 /* 3195 * The kernel and userspace may race to initiate a scrub, but 3196 * the scrub thread is prepared to lose that initial race. It 3197 * just needs guarantees that any ars it initiates are not 3198 * interrupted by any intervening start reqeusts from userspace. 3199 */ 3200 if (work_busy(&acpi_desc->work)) 3201 return -EBUSY; 3202 3203 return 0; 3204 } 3205 3206 int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc, u8 flags) 3207 { 3208 struct device *dev = acpi_desc->dev; 3209 struct nfit_spa *nfit_spa; 3210 3211 if (work_busy(&acpi_desc->work)) 3212 return -EBUSY; 3213 3214 mutex_lock(&acpi_desc->init_mutex); 3215 if (acpi_desc->cancel) { 3216 mutex_unlock(&acpi_desc->init_mutex); 3217 return 0; 3218 } 3219 3220 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 3221 struct acpi_nfit_system_address *spa = nfit_spa->spa; 3222 3223 if (nfit_spa_type(spa) != NFIT_SPA_PM) 3224 continue; 3225 3226 nfit_spa->ars_required = 1; 3227 } 3228 acpi_desc->ars_start_flags = flags; 3229 queue_work(nfit_wq, &acpi_desc->work); 3230 dev_dbg(dev, "%s: ars_scan triggered\n", __func__); 3231 mutex_unlock(&acpi_desc->init_mutex); 3232 3233 return 0; 3234 } 3235 3236 void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev) 3237 { 3238 struct nvdimm_bus_descriptor *nd_desc; 3239 3240 dev_set_drvdata(dev, acpi_desc); 3241 acpi_desc->dev = dev; 3242 acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io; 3243 nd_desc = &acpi_desc->nd_desc; 3244 nd_desc->provider_name = "ACPI.NFIT"; 3245 nd_desc->module = THIS_MODULE; 3246 nd_desc->ndctl = acpi_nfit_ctl; 3247 nd_desc->flush_probe = acpi_nfit_flush_probe; 3248 nd_desc->clear_to_send = acpi_nfit_clear_to_send; 3249 nd_desc->attr_groups = acpi_nfit_attribute_groups; 3250 3251 INIT_LIST_HEAD(&acpi_desc->spas); 3252 INIT_LIST_HEAD(&acpi_desc->dcrs); 3253 INIT_LIST_HEAD(&acpi_desc->bdws); 3254 INIT_LIST_HEAD(&acpi_desc->idts); 3255 INIT_LIST_HEAD(&acpi_desc->flushes); 3256 INIT_LIST_HEAD(&acpi_desc->memdevs); 3257 INIT_LIST_HEAD(&acpi_desc->dimms); 3258 INIT_LIST_HEAD(&acpi_desc->list); 3259 mutex_init(&acpi_desc->init_mutex); 3260 INIT_WORK(&acpi_desc->work, acpi_nfit_scrub); 3261 } 3262 EXPORT_SYMBOL_GPL(acpi_nfit_desc_init); 3263 3264 static void acpi_nfit_put_table(void *table) 3265 { 3266 acpi_put_table(table); 3267 } 3268 3269 void acpi_nfit_shutdown(void *data) 3270 { 3271 struct acpi_nfit_desc *acpi_desc = data; 3272 struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); 3273 3274 /* 3275 * Destruct under acpi_desc_lock so that nfit_handle_mce does not 3276 * race teardown 3277 */ 3278 mutex_lock(&acpi_desc_lock); 3279 list_del(&acpi_desc->list); 3280 mutex_unlock(&acpi_desc_lock); 3281 3282 mutex_lock(&acpi_desc->init_mutex); 3283 acpi_desc->cancel = 1; 3284 mutex_unlock(&acpi_desc->init_mutex); 3285 3286 /* 3287 * Bounce the nvdimm bus lock to make sure any in-flight 3288 * acpi_nfit_ars_rescan() submissions have had a chance to 3289 * either submit or see ->cancel set. 3290 */ 3291 device_lock(bus_dev); 3292 device_unlock(bus_dev); 3293 3294 flush_workqueue(nfit_wq); 3295 } 3296 EXPORT_SYMBOL_GPL(acpi_nfit_shutdown); 3297 3298 static int acpi_nfit_add(struct acpi_device *adev) 3299 { 3300 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 3301 struct acpi_nfit_desc *acpi_desc; 3302 struct device *dev = &adev->dev; 3303 struct acpi_table_header *tbl; 3304 acpi_status status = AE_OK; 3305 acpi_size sz; 3306 int rc = 0; 3307 3308 status = acpi_get_table(ACPI_SIG_NFIT, 0, &tbl); 3309 if (ACPI_FAILURE(status)) { 3310 /* This is ok, we could have an nvdimm hotplugged later */ 3311 dev_dbg(dev, "failed to find NFIT at startup\n"); 3312 return 0; 3313 } 3314 3315 rc = devm_add_action_or_reset(dev, acpi_nfit_put_table, tbl); 3316 if (rc) 3317 return rc; 3318 sz = tbl->length; 3319 3320 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL); 3321 if (!acpi_desc) 3322 return -ENOMEM; 3323 acpi_nfit_desc_init(acpi_desc, &adev->dev); 3324 3325 /* Save the acpi header for exporting the revision via sysfs */ 3326 acpi_desc->acpi_header = *tbl; 3327 3328 /* Evaluate _FIT and override with that if present */ 3329 status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf); 3330 if (ACPI_SUCCESS(status) && buf.length > 0) { 3331 union acpi_object *obj = buf.pointer; 3332 3333 if (obj->type == ACPI_TYPE_BUFFER) 3334 rc = acpi_nfit_init(acpi_desc, obj->buffer.pointer, 3335 obj->buffer.length); 3336 else 3337 dev_dbg(dev, "%s invalid type %d, ignoring _FIT\n", 3338 __func__, (int) obj->type); 3339 kfree(buf.pointer); 3340 } else 3341 /* skip over the lead-in header table */ 3342 rc = acpi_nfit_init(acpi_desc, (void *) tbl 3343 + sizeof(struct acpi_table_nfit), 3344 sz - sizeof(struct acpi_table_nfit)); 3345 3346 if (rc) 3347 return rc; 3348 return devm_add_action_or_reset(dev, acpi_nfit_shutdown, acpi_desc); 3349 } 3350 3351 static int acpi_nfit_remove(struct acpi_device *adev) 3352 { 3353 /* see acpi_nfit_unregister */ 3354 return 0; 3355 } 3356 3357 static void acpi_nfit_update_notify(struct device *dev, acpi_handle handle) 3358 { 3359 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev); 3360 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 3361 union acpi_object *obj; 3362 acpi_status status; 3363 int ret; 3364 3365 if (!dev->driver) { 3366 /* dev->driver may be null if we're being removed */ 3367 dev_dbg(dev, "%s: no driver found for dev\n", __func__); 3368 return; 3369 } 3370 3371 if (!acpi_desc) { 3372 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL); 3373 if (!acpi_desc) 3374 return; 3375 acpi_nfit_desc_init(acpi_desc, dev); 3376 } else { 3377 /* 3378 * Finish previous registration before considering new 3379 * regions. 3380 */ 3381 flush_workqueue(nfit_wq); 3382 } 3383 3384 /* Evaluate _FIT */ 3385 status = acpi_evaluate_object(handle, "_FIT", NULL, &buf); 3386 if (ACPI_FAILURE(status)) { 3387 dev_err(dev, "failed to evaluate _FIT\n"); 3388 return; 3389 } 3390 3391 obj = buf.pointer; 3392 if (obj->type == ACPI_TYPE_BUFFER) { 3393 ret = acpi_nfit_init(acpi_desc, obj->buffer.pointer, 3394 obj->buffer.length); 3395 if (ret) 3396 dev_err(dev, "failed to merge updated NFIT\n"); 3397 } else 3398 dev_err(dev, "Invalid _FIT\n"); 3399 kfree(buf.pointer); 3400 } 3401 3402 static void acpi_nfit_uc_error_notify(struct device *dev, acpi_handle handle) 3403 { 3404 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev); 3405 u8 flags = (acpi_desc->scrub_mode == HW_ERROR_SCRUB_ON) ? 3406 0 : ND_ARS_RETURN_PREV_DATA; 3407 3408 acpi_nfit_ars_rescan(acpi_desc, flags); 3409 } 3410 3411 void __acpi_nfit_notify(struct device *dev, acpi_handle handle, u32 event) 3412 { 3413 dev_dbg(dev, "%s: event: 0x%x\n", __func__, event); 3414 3415 switch (event) { 3416 case NFIT_NOTIFY_UPDATE: 3417 return acpi_nfit_update_notify(dev, handle); 3418 case NFIT_NOTIFY_UC_MEMORY_ERROR: 3419 return acpi_nfit_uc_error_notify(dev, handle); 3420 default: 3421 return; 3422 } 3423 } 3424 EXPORT_SYMBOL_GPL(__acpi_nfit_notify); 3425 3426 static void acpi_nfit_notify(struct acpi_device *adev, u32 event) 3427 { 3428 device_lock(&adev->dev); 3429 __acpi_nfit_notify(&adev->dev, adev->handle, event); 3430 device_unlock(&adev->dev); 3431 } 3432 3433 static const struct acpi_device_id acpi_nfit_ids[] = { 3434 { "ACPI0012", 0 }, 3435 { "", 0 }, 3436 }; 3437 MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids); 3438 3439 static struct acpi_driver acpi_nfit_driver = { 3440 .name = KBUILD_MODNAME, 3441 .ids = acpi_nfit_ids, 3442 .ops = { 3443 .add = acpi_nfit_add, 3444 .remove = acpi_nfit_remove, 3445 .notify = acpi_nfit_notify, 3446 }, 3447 }; 3448 3449 static __init int nfit_init(void) 3450 { 3451 int ret; 3452 3453 BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40); 3454 BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56); 3455 BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48); 3456 BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20); 3457 BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9); 3458 BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80); 3459 BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40); 3460 3461 guid_parse(UUID_VOLATILE_MEMORY, &nfit_uuid[NFIT_SPA_VOLATILE]); 3462 guid_parse(UUID_PERSISTENT_MEMORY, &nfit_uuid[NFIT_SPA_PM]); 3463 guid_parse(UUID_CONTROL_REGION, &nfit_uuid[NFIT_SPA_DCR]); 3464 guid_parse(UUID_DATA_REGION, &nfit_uuid[NFIT_SPA_BDW]); 3465 guid_parse(UUID_VOLATILE_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_VDISK]); 3466 guid_parse(UUID_VOLATILE_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_VCD]); 3467 guid_parse(UUID_PERSISTENT_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_PDISK]); 3468 guid_parse(UUID_PERSISTENT_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_PCD]); 3469 guid_parse(UUID_NFIT_BUS, &nfit_uuid[NFIT_DEV_BUS]); 3470 guid_parse(UUID_NFIT_DIMM, &nfit_uuid[NFIT_DEV_DIMM]); 3471 guid_parse(UUID_NFIT_DIMM_N_HPE1, &nfit_uuid[NFIT_DEV_DIMM_N_HPE1]); 3472 guid_parse(UUID_NFIT_DIMM_N_HPE2, &nfit_uuid[NFIT_DEV_DIMM_N_HPE2]); 3473 guid_parse(UUID_NFIT_DIMM_N_MSFT, &nfit_uuid[NFIT_DEV_DIMM_N_MSFT]); 3474 3475 nfit_wq = create_singlethread_workqueue("nfit"); 3476 if (!nfit_wq) 3477 return -ENOMEM; 3478 3479 nfit_mce_register(); 3480 ret = acpi_bus_register_driver(&acpi_nfit_driver); 3481 if (ret) { 3482 nfit_mce_unregister(); 3483 destroy_workqueue(nfit_wq); 3484 } 3485 3486 return ret; 3487 3488 } 3489 3490 static __exit void nfit_exit(void) 3491 { 3492 nfit_mce_unregister(); 3493 acpi_bus_unregister_driver(&acpi_nfit_driver); 3494 destroy_workqueue(nfit_wq); 3495 WARN_ON(!list_empty(&acpi_descs)); 3496 } 3497 3498 module_init(nfit_init); 3499 module_exit(nfit_exit); 3500 MODULE_LICENSE("GPL v2"); 3501 MODULE_AUTHOR("Intel Corporation"); 3502