1 /* 2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of version 2 of the GNU General Public License as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 */ 13 #include <linux/list_sort.h> 14 #include <linux/libnvdimm.h> 15 #include <linux/module.h> 16 #include <linux/mutex.h> 17 #include <linux/ndctl.h> 18 #include <linux/sysfs.h> 19 #include <linux/delay.h> 20 #include <linux/list.h> 21 #include <linux/acpi.h> 22 #include <linux/sort.h> 23 #include <linux/io.h> 24 #include <linux/nd.h> 25 #include <asm/cacheflush.h> 26 #include <acpi/nfit.h> 27 #include "nfit.h" 28 29 /* 30 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is 31 * irrelevant. 32 */ 33 #include <linux/io-64-nonatomic-hi-lo.h> 34 35 static bool force_enable_dimms; 36 module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR); 37 MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status"); 38 39 static unsigned int scrub_timeout = NFIT_ARS_TIMEOUT; 40 module_param(scrub_timeout, uint, S_IRUGO|S_IWUSR); 41 MODULE_PARM_DESC(scrub_timeout, "Initial scrub timeout in seconds"); 42 43 /* after three payloads of overflow, it's dead jim */ 44 static unsigned int scrub_overflow_abort = 3; 45 module_param(scrub_overflow_abort, uint, S_IRUGO|S_IWUSR); 46 MODULE_PARM_DESC(scrub_overflow_abort, 47 "Number of times we overflow ARS results before abort"); 48 49 static bool disable_vendor_specific; 50 module_param(disable_vendor_specific, bool, S_IRUGO); 51 MODULE_PARM_DESC(disable_vendor_specific, 52 "Limit commands to the publicly specified set"); 53 54 static unsigned long override_dsm_mask; 55 module_param(override_dsm_mask, ulong, S_IRUGO); 56 MODULE_PARM_DESC(override_dsm_mask, "Bitmask of allowed NVDIMM DSM functions"); 57 58 static int default_dsm_family = -1; 59 module_param(default_dsm_family, int, S_IRUGO); 60 MODULE_PARM_DESC(default_dsm_family, 61 "Try this DSM type first when identifying NVDIMM family"); 62 63 LIST_HEAD(acpi_descs); 64 DEFINE_MUTEX(acpi_desc_lock); 65 66 static struct workqueue_struct *nfit_wq; 67 68 struct nfit_table_prev { 69 struct list_head spas; 70 struct list_head memdevs; 71 struct list_head dcrs; 72 struct list_head bdws; 73 struct list_head idts; 74 struct list_head flushes; 75 }; 76 77 static guid_t nfit_uuid[NFIT_UUID_MAX]; 78 79 const guid_t *to_nfit_uuid(enum nfit_uuids id) 80 { 81 return &nfit_uuid[id]; 82 } 83 EXPORT_SYMBOL(to_nfit_uuid); 84 85 static struct acpi_nfit_desc *to_acpi_nfit_desc( 86 struct nvdimm_bus_descriptor *nd_desc) 87 { 88 return container_of(nd_desc, struct acpi_nfit_desc, nd_desc); 89 } 90 91 static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc) 92 { 93 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 94 95 /* 96 * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct 97 * acpi_device. 98 */ 99 if (!nd_desc->provider_name 100 || strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0) 101 return NULL; 102 103 return to_acpi_device(acpi_desc->dev); 104 } 105 106 static int xlat_bus_status(void *buf, unsigned int cmd, u32 status) 107 { 108 struct nd_cmd_clear_error *clear_err; 109 struct nd_cmd_ars_status *ars_status; 110 u16 flags; 111 112 switch (cmd) { 113 case ND_CMD_ARS_CAP: 114 if ((status & 0xffff) == NFIT_ARS_CAP_NONE) 115 return -ENOTTY; 116 117 /* Command failed */ 118 if (status & 0xffff) 119 return -EIO; 120 121 /* No supported scan types for this range */ 122 flags = ND_ARS_PERSISTENT | ND_ARS_VOLATILE; 123 if ((status >> 16 & flags) == 0) 124 return -ENOTTY; 125 return 0; 126 case ND_CMD_ARS_START: 127 /* ARS is in progress */ 128 if ((status & 0xffff) == NFIT_ARS_START_BUSY) 129 return -EBUSY; 130 131 /* Command failed */ 132 if (status & 0xffff) 133 return -EIO; 134 return 0; 135 case ND_CMD_ARS_STATUS: 136 ars_status = buf; 137 /* Command failed */ 138 if (status & 0xffff) 139 return -EIO; 140 /* Check extended status (Upper two bytes) */ 141 if (status == NFIT_ARS_STATUS_DONE) 142 return 0; 143 144 /* ARS is in progress */ 145 if (status == NFIT_ARS_STATUS_BUSY) 146 return -EBUSY; 147 148 /* No ARS performed for the current boot */ 149 if (status == NFIT_ARS_STATUS_NONE) 150 return -EAGAIN; 151 152 /* 153 * ARS interrupted, either we overflowed or some other 154 * agent wants the scan to stop. If we didn't overflow 155 * then just continue with the returned results. 156 */ 157 if (status == NFIT_ARS_STATUS_INTR) { 158 if (ars_status->out_length >= 40 && (ars_status->flags 159 & NFIT_ARS_F_OVERFLOW)) 160 return -ENOSPC; 161 return 0; 162 } 163 164 /* Unknown status */ 165 if (status >> 16) 166 return -EIO; 167 return 0; 168 case ND_CMD_CLEAR_ERROR: 169 clear_err = buf; 170 if (status & 0xffff) 171 return -EIO; 172 if (!clear_err->cleared) 173 return -EIO; 174 if (clear_err->length > clear_err->cleared) 175 return clear_err->cleared; 176 return 0; 177 default: 178 break; 179 } 180 181 /* all other non-zero status results in an error */ 182 if (status) 183 return -EIO; 184 return 0; 185 } 186 187 #define ACPI_LABELS_LOCKED 3 188 189 static int xlat_nvdimm_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd, 190 u32 status) 191 { 192 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 193 194 switch (cmd) { 195 case ND_CMD_GET_CONFIG_SIZE: 196 /* 197 * In the _LSI, _LSR, _LSW case the locked status is 198 * communicated via the read/write commands 199 */ 200 if (nfit_mem->has_lsi) 201 break; 202 203 if (status >> 16 & ND_CONFIG_LOCKED) 204 return -EACCES; 205 break; 206 case ND_CMD_GET_CONFIG_DATA: 207 if (nfit_mem->has_lsr && status == ACPI_LABELS_LOCKED) 208 return -EACCES; 209 break; 210 case ND_CMD_SET_CONFIG_DATA: 211 if (nfit_mem->has_lsw && status == ACPI_LABELS_LOCKED) 212 return -EACCES; 213 break; 214 default: 215 break; 216 } 217 218 /* all other non-zero status results in an error */ 219 if (status) 220 return -EIO; 221 return 0; 222 } 223 224 static int xlat_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd, 225 u32 status) 226 { 227 if (!nvdimm) 228 return xlat_bus_status(buf, cmd, status); 229 return xlat_nvdimm_status(nvdimm, buf, cmd, status); 230 } 231 232 /* convert _LS{I,R} packages to the buffer object acpi_nfit_ctl expects */ 233 static union acpi_object *pkg_to_buf(union acpi_object *pkg) 234 { 235 int i; 236 void *dst; 237 size_t size = 0; 238 union acpi_object *buf = NULL; 239 240 if (pkg->type != ACPI_TYPE_PACKAGE) { 241 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n", 242 pkg->type); 243 goto err; 244 } 245 246 for (i = 0; i < pkg->package.count; i++) { 247 union acpi_object *obj = &pkg->package.elements[i]; 248 249 if (obj->type == ACPI_TYPE_INTEGER) 250 size += 4; 251 else if (obj->type == ACPI_TYPE_BUFFER) 252 size += obj->buffer.length; 253 else { 254 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n", 255 obj->type); 256 goto err; 257 } 258 } 259 260 buf = ACPI_ALLOCATE(sizeof(*buf) + size); 261 if (!buf) 262 goto err; 263 264 dst = buf + 1; 265 buf->type = ACPI_TYPE_BUFFER; 266 buf->buffer.length = size; 267 buf->buffer.pointer = dst; 268 for (i = 0; i < pkg->package.count; i++) { 269 union acpi_object *obj = &pkg->package.elements[i]; 270 271 if (obj->type == ACPI_TYPE_INTEGER) { 272 memcpy(dst, &obj->integer.value, 4); 273 dst += 4; 274 } else if (obj->type == ACPI_TYPE_BUFFER) { 275 memcpy(dst, obj->buffer.pointer, obj->buffer.length); 276 dst += obj->buffer.length; 277 } 278 } 279 err: 280 ACPI_FREE(pkg); 281 return buf; 282 } 283 284 static union acpi_object *int_to_buf(union acpi_object *integer) 285 { 286 union acpi_object *buf = ACPI_ALLOCATE(sizeof(*buf) + 4); 287 void *dst = NULL; 288 289 if (!buf) 290 goto err; 291 292 if (integer->type != ACPI_TYPE_INTEGER) { 293 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n", 294 integer->type); 295 goto err; 296 } 297 298 dst = buf + 1; 299 buf->type = ACPI_TYPE_BUFFER; 300 buf->buffer.length = 4; 301 buf->buffer.pointer = dst; 302 memcpy(dst, &integer->integer.value, 4); 303 err: 304 ACPI_FREE(integer); 305 return buf; 306 } 307 308 static union acpi_object *acpi_label_write(acpi_handle handle, u32 offset, 309 u32 len, void *data) 310 { 311 acpi_status rc; 312 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 313 struct acpi_object_list input = { 314 .count = 3, 315 .pointer = (union acpi_object []) { 316 [0] = { 317 .integer.type = ACPI_TYPE_INTEGER, 318 .integer.value = offset, 319 }, 320 [1] = { 321 .integer.type = ACPI_TYPE_INTEGER, 322 .integer.value = len, 323 }, 324 [2] = { 325 .buffer.type = ACPI_TYPE_BUFFER, 326 .buffer.pointer = data, 327 .buffer.length = len, 328 }, 329 }, 330 }; 331 332 rc = acpi_evaluate_object(handle, "_LSW", &input, &buf); 333 if (ACPI_FAILURE(rc)) 334 return NULL; 335 return int_to_buf(buf.pointer); 336 } 337 338 static union acpi_object *acpi_label_read(acpi_handle handle, u32 offset, 339 u32 len) 340 { 341 acpi_status rc; 342 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 343 struct acpi_object_list input = { 344 .count = 2, 345 .pointer = (union acpi_object []) { 346 [0] = { 347 .integer.type = ACPI_TYPE_INTEGER, 348 .integer.value = offset, 349 }, 350 [1] = { 351 .integer.type = ACPI_TYPE_INTEGER, 352 .integer.value = len, 353 }, 354 }, 355 }; 356 357 rc = acpi_evaluate_object(handle, "_LSR", &input, &buf); 358 if (ACPI_FAILURE(rc)) 359 return NULL; 360 return pkg_to_buf(buf.pointer); 361 } 362 363 static union acpi_object *acpi_label_info(acpi_handle handle) 364 { 365 acpi_status rc; 366 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 367 368 rc = acpi_evaluate_object(handle, "_LSI", NULL, &buf); 369 if (ACPI_FAILURE(rc)) 370 return NULL; 371 return pkg_to_buf(buf.pointer); 372 } 373 374 static u8 nfit_dsm_revid(unsigned family, unsigned func) 375 { 376 static const u8 revid_table[NVDIMM_FAMILY_MAX+1][32] = { 377 [NVDIMM_FAMILY_INTEL] = { 378 [NVDIMM_INTEL_GET_MODES] = 2, 379 [NVDIMM_INTEL_GET_FWINFO] = 2, 380 [NVDIMM_INTEL_START_FWUPDATE] = 2, 381 [NVDIMM_INTEL_SEND_FWUPDATE] = 2, 382 [NVDIMM_INTEL_FINISH_FWUPDATE] = 2, 383 [NVDIMM_INTEL_QUERY_FWUPDATE] = 2, 384 [NVDIMM_INTEL_SET_THRESHOLD] = 2, 385 [NVDIMM_INTEL_INJECT_ERROR] = 2, 386 }, 387 }; 388 u8 id; 389 390 if (family > NVDIMM_FAMILY_MAX) 391 return 0; 392 if (func > 31) 393 return 0; 394 id = revid_table[family][func]; 395 if (id == 0) 396 return 1; /* default */ 397 return id; 398 } 399 400 int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm, 401 unsigned int cmd, void *buf, unsigned int buf_len, int *cmd_rc) 402 { 403 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 404 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 405 union acpi_object in_obj, in_buf, *out_obj; 406 const struct nd_cmd_desc *desc = NULL; 407 struct device *dev = acpi_desc->dev; 408 struct nd_cmd_pkg *call_pkg = NULL; 409 const char *cmd_name, *dimm_name; 410 unsigned long cmd_mask, dsm_mask; 411 u32 offset, fw_status = 0; 412 acpi_handle handle; 413 unsigned int func; 414 const guid_t *guid; 415 int rc, i; 416 417 func = cmd; 418 if (cmd == ND_CMD_CALL) { 419 call_pkg = buf; 420 func = call_pkg->nd_command; 421 422 for (i = 0; i < ARRAY_SIZE(call_pkg->nd_reserved2); i++) 423 if (call_pkg->nd_reserved2[i]) 424 return -EINVAL; 425 } 426 427 if (nvdimm) { 428 struct acpi_device *adev = nfit_mem->adev; 429 430 if (!adev) 431 return -ENOTTY; 432 if (call_pkg && nfit_mem->family != call_pkg->nd_family) 433 return -ENOTTY; 434 435 dimm_name = nvdimm_name(nvdimm); 436 cmd_name = nvdimm_cmd_name(cmd); 437 cmd_mask = nvdimm_cmd_mask(nvdimm); 438 dsm_mask = nfit_mem->dsm_mask; 439 desc = nd_cmd_dimm_desc(cmd); 440 guid = to_nfit_uuid(nfit_mem->family); 441 handle = adev->handle; 442 } else { 443 struct acpi_device *adev = to_acpi_dev(acpi_desc); 444 445 cmd_name = nvdimm_bus_cmd_name(cmd); 446 cmd_mask = nd_desc->cmd_mask; 447 dsm_mask = cmd_mask; 448 if (cmd == ND_CMD_CALL) 449 dsm_mask = nd_desc->bus_dsm_mask; 450 desc = nd_cmd_bus_desc(cmd); 451 guid = to_nfit_uuid(NFIT_DEV_BUS); 452 handle = adev->handle; 453 dimm_name = "bus"; 454 } 455 456 if (!desc || (cmd && (desc->out_num + desc->in_num == 0))) 457 return -ENOTTY; 458 459 if (!test_bit(cmd, &cmd_mask) || !test_bit(func, &dsm_mask)) 460 return -ENOTTY; 461 462 in_obj.type = ACPI_TYPE_PACKAGE; 463 in_obj.package.count = 1; 464 in_obj.package.elements = &in_buf; 465 in_buf.type = ACPI_TYPE_BUFFER; 466 in_buf.buffer.pointer = buf; 467 in_buf.buffer.length = 0; 468 469 /* libnvdimm has already validated the input envelope */ 470 for (i = 0; i < desc->in_num; i++) 471 in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc, 472 i, buf); 473 474 if (call_pkg) { 475 /* skip over package wrapper */ 476 in_buf.buffer.pointer = (void *) &call_pkg->nd_payload; 477 in_buf.buffer.length = call_pkg->nd_size_in; 478 } 479 480 dev_dbg(dev, "%s:%s cmd: %d: func: %d input length: %d\n", 481 __func__, dimm_name, cmd, func, in_buf.buffer.length); 482 print_hex_dump_debug("nvdimm in ", DUMP_PREFIX_OFFSET, 4, 4, 483 in_buf.buffer.pointer, 484 min_t(u32, 256, in_buf.buffer.length), true); 485 486 /* call the BIOS, prefer the named methods over _DSM if available */ 487 if (nvdimm && cmd == ND_CMD_GET_CONFIG_SIZE && nfit_mem->has_lsi) 488 out_obj = acpi_label_info(handle); 489 else if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && nfit_mem->has_lsr) { 490 struct nd_cmd_get_config_data_hdr *p = buf; 491 492 out_obj = acpi_label_read(handle, p->in_offset, p->in_length); 493 } else if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA 494 && nfit_mem->has_lsw) { 495 struct nd_cmd_set_config_hdr *p = buf; 496 497 out_obj = acpi_label_write(handle, p->in_offset, p->in_length, 498 p->in_buf); 499 } else { 500 u8 revid; 501 502 if (nvdimm) 503 revid = nfit_dsm_revid(nfit_mem->family, func); 504 else 505 revid = 1; 506 out_obj = acpi_evaluate_dsm(handle, guid, revid, func, &in_obj); 507 } 508 509 if (!out_obj) { 510 dev_dbg(dev, "%s:%s _DSM failed cmd: %s\n", __func__, dimm_name, 511 cmd_name); 512 return -EINVAL; 513 } 514 515 if (call_pkg) { 516 call_pkg->nd_fw_size = out_obj->buffer.length; 517 memcpy(call_pkg->nd_payload + call_pkg->nd_size_in, 518 out_obj->buffer.pointer, 519 min(call_pkg->nd_fw_size, call_pkg->nd_size_out)); 520 521 ACPI_FREE(out_obj); 522 /* 523 * Need to support FW function w/o known size in advance. 524 * Caller can determine required size based upon nd_fw_size. 525 * If we return an error (like elsewhere) then caller wouldn't 526 * be able to rely upon data returned to make calculation. 527 */ 528 return 0; 529 } 530 531 if (out_obj->package.type != ACPI_TYPE_BUFFER) { 532 dev_dbg(dev, "%s:%s unexpected output object type cmd: %s type: %d\n", 533 __func__, dimm_name, cmd_name, out_obj->type); 534 rc = -EINVAL; 535 goto out; 536 } 537 538 dev_dbg(dev, "%s:%s cmd: %s output length: %d\n", __func__, dimm_name, 539 cmd_name, out_obj->buffer.length); 540 print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4, 4, 541 out_obj->buffer.pointer, 542 min_t(u32, 128, out_obj->buffer.length), true); 543 544 for (i = 0, offset = 0; i < desc->out_num; i++) { 545 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf, 546 (u32 *) out_obj->buffer.pointer, 547 out_obj->buffer.length - offset); 548 549 if (offset + out_size > out_obj->buffer.length) { 550 dev_dbg(dev, "%s:%s output object underflow cmd: %s field: %d\n", 551 __func__, dimm_name, cmd_name, i); 552 break; 553 } 554 555 if (in_buf.buffer.length + offset + out_size > buf_len) { 556 dev_dbg(dev, "%s:%s output overrun cmd: %s field: %d\n", 557 __func__, dimm_name, cmd_name, i); 558 rc = -ENXIO; 559 goto out; 560 } 561 memcpy(buf + in_buf.buffer.length + offset, 562 out_obj->buffer.pointer + offset, out_size); 563 offset += out_size; 564 } 565 566 /* 567 * Set fw_status for all the commands with a known format to be 568 * later interpreted by xlat_status(). 569 */ 570 if (i >= 1 && ((!nvdimm && cmd >= ND_CMD_ARS_CAP 571 && cmd <= ND_CMD_CLEAR_ERROR) 572 || (nvdimm && cmd >= ND_CMD_SMART 573 && cmd <= ND_CMD_VENDOR))) 574 fw_status = *(u32 *) out_obj->buffer.pointer; 575 576 if (offset + in_buf.buffer.length < buf_len) { 577 if (i >= 1) { 578 /* 579 * status valid, return the number of bytes left 580 * unfilled in the output buffer 581 */ 582 rc = buf_len - offset - in_buf.buffer.length; 583 if (cmd_rc) 584 *cmd_rc = xlat_status(nvdimm, buf, cmd, 585 fw_status); 586 } else { 587 dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n", 588 __func__, dimm_name, cmd_name, buf_len, 589 offset); 590 rc = -ENXIO; 591 } 592 } else { 593 rc = 0; 594 if (cmd_rc) 595 *cmd_rc = xlat_status(nvdimm, buf, cmd, fw_status); 596 } 597 598 out: 599 ACPI_FREE(out_obj); 600 601 return rc; 602 } 603 EXPORT_SYMBOL_GPL(acpi_nfit_ctl); 604 605 static const char *spa_type_name(u16 type) 606 { 607 static const char *to_name[] = { 608 [NFIT_SPA_VOLATILE] = "volatile", 609 [NFIT_SPA_PM] = "pmem", 610 [NFIT_SPA_DCR] = "dimm-control-region", 611 [NFIT_SPA_BDW] = "block-data-window", 612 [NFIT_SPA_VDISK] = "volatile-disk", 613 [NFIT_SPA_VCD] = "volatile-cd", 614 [NFIT_SPA_PDISK] = "persistent-disk", 615 [NFIT_SPA_PCD] = "persistent-cd", 616 617 }; 618 619 if (type > NFIT_SPA_PCD) 620 return "unknown"; 621 622 return to_name[type]; 623 } 624 625 int nfit_spa_type(struct acpi_nfit_system_address *spa) 626 { 627 int i; 628 629 for (i = 0; i < NFIT_UUID_MAX; i++) 630 if (guid_equal(to_nfit_uuid(i), (guid_t *)&spa->range_guid)) 631 return i; 632 return -1; 633 } 634 635 static bool add_spa(struct acpi_nfit_desc *acpi_desc, 636 struct nfit_table_prev *prev, 637 struct acpi_nfit_system_address *spa) 638 { 639 struct device *dev = acpi_desc->dev; 640 struct nfit_spa *nfit_spa; 641 642 if (spa->header.length != sizeof(*spa)) 643 return false; 644 645 list_for_each_entry(nfit_spa, &prev->spas, list) { 646 if (memcmp(nfit_spa->spa, spa, sizeof(*spa)) == 0) { 647 list_move_tail(&nfit_spa->list, &acpi_desc->spas); 648 return true; 649 } 650 } 651 652 nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa) + sizeof(*spa), 653 GFP_KERNEL); 654 if (!nfit_spa) 655 return false; 656 INIT_LIST_HEAD(&nfit_spa->list); 657 memcpy(nfit_spa->spa, spa, sizeof(*spa)); 658 list_add_tail(&nfit_spa->list, &acpi_desc->spas); 659 dev_dbg(dev, "%s: spa index: %d type: %s\n", __func__, 660 spa->range_index, 661 spa_type_name(nfit_spa_type(spa))); 662 return true; 663 } 664 665 static bool add_memdev(struct acpi_nfit_desc *acpi_desc, 666 struct nfit_table_prev *prev, 667 struct acpi_nfit_memory_map *memdev) 668 { 669 struct device *dev = acpi_desc->dev; 670 struct nfit_memdev *nfit_memdev; 671 672 if (memdev->header.length != sizeof(*memdev)) 673 return false; 674 675 list_for_each_entry(nfit_memdev, &prev->memdevs, list) 676 if (memcmp(nfit_memdev->memdev, memdev, sizeof(*memdev)) == 0) { 677 list_move_tail(&nfit_memdev->list, &acpi_desc->memdevs); 678 return true; 679 } 680 681 nfit_memdev = devm_kzalloc(dev, sizeof(*nfit_memdev) + sizeof(*memdev), 682 GFP_KERNEL); 683 if (!nfit_memdev) 684 return false; 685 INIT_LIST_HEAD(&nfit_memdev->list); 686 memcpy(nfit_memdev->memdev, memdev, sizeof(*memdev)); 687 list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs); 688 dev_dbg(dev, "%s: memdev handle: %#x spa: %d dcr: %d flags: %#x\n", 689 __func__, memdev->device_handle, memdev->range_index, 690 memdev->region_index, memdev->flags); 691 return true; 692 } 693 694 int nfit_get_smbios_id(u32 device_handle, u16 *flags) 695 { 696 struct acpi_nfit_memory_map *memdev; 697 struct acpi_nfit_desc *acpi_desc; 698 struct nfit_mem *nfit_mem; 699 700 mutex_lock(&acpi_desc_lock); 701 list_for_each_entry(acpi_desc, &acpi_descs, list) { 702 mutex_lock(&acpi_desc->init_mutex); 703 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 704 memdev = __to_nfit_memdev(nfit_mem); 705 if (memdev->device_handle == device_handle) { 706 mutex_unlock(&acpi_desc->init_mutex); 707 mutex_unlock(&acpi_desc_lock); 708 *flags = memdev->flags; 709 return memdev->physical_id; 710 } 711 } 712 mutex_unlock(&acpi_desc->init_mutex); 713 } 714 mutex_unlock(&acpi_desc_lock); 715 716 return -ENODEV; 717 } 718 EXPORT_SYMBOL_GPL(nfit_get_smbios_id); 719 720 /* 721 * An implementation may provide a truncated control region if no block windows 722 * are defined. 723 */ 724 static size_t sizeof_dcr(struct acpi_nfit_control_region *dcr) 725 { 726 if (dcr->header.length < offsetof(struct acpi_nfit_control_region, 727 window_size)) 728 return 0; 729 if (dcr->windows) 730 return sizeof(*dcr); 731 return offsetof(struct acpi_nfit_control_region, window_size); 732 } 733 734 static bool add_dcr(struct acpi_nfit_desc *acpi_desc, 735 struct nfit_table_prev *prev, 736 struct acpi_nfit_control_region *dcr) 737 { 738 struct device *dev = acpi_desc->dev; 739 struct nfit_dcr *nfit_dcr; 740 741 if (!sizeof_dcr(dcr)) 742 return false; 743 744 list_for_each_entry(nfit_dcr, &prev->dcrs, list) 745 if (memcmp(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)) == 0) { 746 list_move_tail(&nfit_dcr->list, &acpi_desc->dcrs); 747 return true; 748 } 749 750 nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr) + sizeof(*dcr), 751 GFP_KERNEL); 752 if (!nfit_dcr) 753 return false; 754 INIT_LIST_HEAD(&nfit_dcr->list); 755 memcpy(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)); 756 list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs); 757 dev_dbg(dev, "%s: dcr index: %d windows: %d\n", __func__, 758 dcr->region_index, dcr->windows); 759 return true; 760 } 761 762 static bool add_bdw(struct acpi_nfit_desc *acpi_desc, 763 struct nfit_table_prev *prev, 764 struct acpi_nfit_data_region *bdw) 765 { 766 struct device *dev = acpi_desc->dev; 767 struct nfit_bdw *nfit_bdw; 768 769 if (bdw->header.length != sizeof(*bdw)) 770 return false; 771 list_for_each_entry(nfit_bdw, &prev->bdws, list) 772 if (memcmp(nfit_bdw->bdw, bdw, sizeof(*bdw)) == 0) { 773 list_move_tail(&nfit_bdw->list, &acpi_desc->bdws); 774 return true; 775 } 776 777 nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw) + sizeof(*bdw), 778 GFP_KERNEL); 779 if (!nfit_bdw) 780 return false; 781 INIT_LIST_HEAD(&nfit_bdw->list); 782 memcpy(nfit_bdw->bdw, bdw, sizeof(*bdw)); 783 list_add_tail(&nfit_bdw->list, &acpi_desc->bdws); 784 dev_dbg(dev, "%s: bdw dcr: %d windows: %d\n", __func__, 785 bdw->region_index, bdw->windows); 786 return true; 787 } 788 789 static size_t sizeof_idt(struct acpi_nfit_interleave *idt) 790 { 791 if (idt->header.length < sizeof(*idt)) 792 return 0; 793 return sizeof(*idt) + sizeof(u32) * (idt->line_count - 1); 794 } 795 796 static bool add_idt(struct acpi_nfit_desc *acpi_desc, 797 struct nfit_table_prev *prev, 798 struct acpi_nfit_interleave *idt) 799 { 800 struct device *dev = acpi_desc->dev; 801 struct nfit_idt *nfit_idt; 802 803 if (!sizeof_idt(idt)) 804 return false; 805 806 list_for_each_entry(nfit_idt, &prev->idts, list) { 807 if (sizeof_idt(nfit_idt->idt) != sizeof_idt(idt)) 808 continue; 809 810 if (memcmp(nfit_idt->idt, idt, sizeof_idt(idt)) == 0) { 811 list_move_tail(&nfit_idt->list, &acpi_desc->idts); 812 return true; 813 } 814 } 815 816 nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt) + sizeof_idt(idt), 817 GFP_KERNEL); 818 if (!nfit_idt) 819 return false; 820 INIT_LIST_HEAD(&nfit_idt->list); 821 memcpy(nfit_idt->idt, idt, sizeof_idt(idt)); 822 list_add_tail(&nfit_idt->list, &acpi_desc->idts); 823 dev_dbg(dev, "%s: idt index: %d num_lines: %d\n", __func__, 824 idt->interleave_index, idt->line_count); 825 return true; 826 } 827 828 static size_t sizeof_flush(struct acpi_nfit_flush_address *flush) 829 { 830 if (flush->header.length < sizeof(*flush)) 831 return 0; 832 return sizeof(*flush) + sizeof(u64) * (flush->hint_count - 1); 833 } 834 835 static bool add_flush(struct acpi_nfit_desc *acpi_desc, 836 struct nfit_table_prev *prev, 837 struct acpi_nfit_flush_address *flush) 838 { 839 struct device *dev = acpi_desc->dev; 840 struct nfit_flush *nfit_flush; 841 842 if (!sizeof_flush(flush)) 843 return false; 844 845 list_for_each_entry(nfit_flush, &prev->flushes, list) { 846 if (sizeof_flush(nfit_flush->flush) != sizeof_flush(flush)) 847 continue; 848 849 if (memcmp(nfit_flush->flush, flush, 850 sizeof_flush(flush)) == 0) { 851 list_move_tail(&nfit_flush->list, &acpi_desc->flushes); 852 return true; 853 } 854 } 855 856 nfit_flush = devm_kzalloc(dev, sizeof(*nfit_flush) 857 + sizeof_flush(flush), GFP_KERNEL); 858 if (!nfit_flush) 859 return false; 860 INIT_LIST_HEAD(&nfit_flush->list); 861 memcpy(nfit_flush->flush, flush, sizeof_flush(flush)); 862 list_add_tail(&nfit_flush->list, &acpi_desc->flushes); 863 dev_dbg(dev, "%s: nfit_flush handle: %d hint_count: %d\n", __func__, 864 flush->device_handle, flush->hint_count); 865 return true; 866 } 867 868 static bool add_platform_cap(struct acpi_nfit_desc *acpi_desc, 869 struct acpi_nfit_capabilities *pcap) 870 { 871 struct device *dev = acpi_desc->dev; 872 u32 mask; 873 874 mask = (1 << (pcap->highest_capability + 1)) - 1; 875 acpi_desc->platform_cap = pcap->capabilities & mask; 876 dev_dbg(dev, "%s: cap: %#x\n", __func__, acpi_desc->platform_cap); 877 return true; 878 } 879 880 static void *add_table(struct acpi_nfit_desc *acpi_desc, 881 struct nfit_table_prev *prev, void *table, const void *end) 882 { 883 struct device *dev = acpi_desc->dev; 884 struct acpi_nfit_header *hdr; 885 void *err = ERR_PTR(-ENOMEM); 886 887 if (table >= end) 888 return NULL; 889 890 hdr = table; 891 if (!hdr->length) { 892 dev_warn(dev, "found a zero length table '%d' parsing nfit\n", 893 hdr->type); 894 return NULL; 895 } 896 897 switch (hdr->type) { 898 case ACPI_NFIT_TYPE_SYSTEM_ADDRESS: 899 if (!add_spa(acpi_desc, prev, table)) 900 return err; 901 break; 902 case ACPI_NFIT_TYPE_MEMORY_MAP: 903 if (!add_memdev(acpi_desc, prev, table)) 904 return err; 905 break; 906 case ACPI_NFIT_TYPE_CONTROL_REGION: 907 if (!add_dcr(acpi_desc, prev, table)) 908 return err; 909 break; 910 case ACPI_NFIT_TYPE_DATA_REGION: 911 if (!add_bdw(acpi_desc, prev, table)) 912 return err; 913 break; 914 case ACPI_NFIT_TYPE_INTERLEAVE: 915 if (!add_idt(acpi_desc, prev, table)) 916 return err; 917 break; 918 case ACPI_NFIT_TYPE_FLUSH_ADDRESS: 919 if (!add_flush(acpi_desc, prev, table)) 920 return err; 921 break; 922 case ACPI_NFIT_TYPE_SMBIOS: 923 dev_dbg(dev, "%s: smbios\n", __func__); 924 break; 925 case ACPI_NFIT_TYPE_CAPABILITIES: 926 if (!add_platform_cap(acpi_desc, table)) 927 return err; 928 break; 929 default: 930 dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type); 931 break; 932 } 933 934 return table + hdr->length; 935 } 936 937 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc, 938 struct nfit_mem *nfit_mem) 939 { 940 u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle; 941 u16 dcr = nfit_mem->dcr->region_index; 942 struct nfit_spa *nfit_spa; 943 944 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 945 u16 range_index = nfit_spa->spa->range_index; 946 int type = nfit_spa_type(nfit_spa->spa); 947 struct nfit_memdev *nfit_memdev; 948 949 if (type != NFIT_SPA_BDW) 950 continue; 951 952 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 953 if (nfit_memdev->memdev->range_index != range_index) 954 continue; 955 if (nfit_memdev->memdev->device_handle != device_handle) 956 continue; 957 if (nfit_memdev->memdev->region_index != dcr) 958 continue; 959 960 nfit_mem->spa_bdw = nfit_spa->spa; 961 return; 962 } 963 } 964 965 dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n", 966 nfit_mem->spa_dcr->range_index); 967 nfit_mem->bdw = NULL; 968 } 969 970 static void nfit_mem_init_bdw(struct acpi_nfit_desc *acpi_desc, 971 struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa) 972 { 973 u16 dcr = __to_nfit_memdev(nfit_mem)->region_index; 974 struct nfit_memdev *nfit_memdev; 975 struct nfit_bdw *nfit_bdw; 976 struct nfit_idt *nfit_idt; 977 u16 idt_idx, range_index; 978 979 list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) { 980 if (nfit_bdw->bdw->region_index != dcr) 981 continue; 982 nfit_mem->bdw = nfit_bdw->bdw; 983 break; 984 } 985 986 if (!nfit_mem->bdw) 987 return; 988 989 nfit_mem_find_spa_bdw(acpi_desc, nfit_mem); 990 991 if (!nfit_mem->spa_bdw) 992 return; 993 994 range_index = nfit_mem->spa_bdw->range_index; 995 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 996 if (nfit_memdev->memdev->range_index != range_index || 997 nfit_memdev->memdev->region_index != dcr) 998 continue; 999 nfit_mem->memdev_bdw = nfit_memdev->memdev; 1000 idt_idx = nfit_memdev->memdev->interleave_index; 1001 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) { 1002 if (nfit_idt->idt->interleave_index != idt_idx) 1003 continue; 1004 nfit_mem->idt_bdw = nfit_idt->idt; 1005 break; 1006 } 1007 break; 1008 } 1009 } 1010 1011 static int __nfit_mem_init(struct acpi_nfit_desc *acpi_desc, 1012 struct acpi_nfit_system_address *spa) 1013 { 1014 struct nfit_mem *nfit_mem, *found; 1015 struct nfit_memdev *nfit_memdev; 1016 int type = spa ? nfit_spa_type(spa) : 0; 1017 1018 switch (type) { 1019 case NFIT_SPA_DCR: 1020 case NFIT_SPA_PM: 1021 break; 1022 default: 1023 if (spa) 1024 return 0; 1025 } 1026 1027 /* 1028 * This loop runs in two modes, when a dimm is mapped the loop 1029 * adds memdev associations to an existing dimm, or creates a 1030 * dimm. In the unmapped dimm case this loop sweeps for memdev 1031 * instances with an invalid / zero range_index and adds those 1032 * dimms without spa associations. 1033 */ 1034 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 1035 struct nfit_flush *nfit_flush; 1036 struct nfit_dcr *nfit_dcr; 1037 u32 device_handle; 1038 u16 dcr; 1039 1040 if (spa && nfit_memdev->memdev->range_index != spa->range_index) 1041 continue; 1042 if (!spa && nfit_memdev->memdev->range_index) 1043 continue; 1044 found = NULL; 1045 dcr = nfit_memdev->memdev->region_index; 1046 device_handle = nfit_memdev->memdev->device_handle; 1047 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) 1048 if (__to_nfit_memdev(nfit_mem)->device_handle 1049 == device_handle) { 1050 found = nfit_mem; 1051 break; 1052 } 1053 1054 if (found) 1055 nfit_mem = found; 1056 else { 1057 nfit_mem = devm_kzalloc(acpi_desc->dev, 1058 sizeof(*nfit_mem), GFP_KERNEL); 1059 if (!nfit_mem) 1060 return -ENOMEM; 1061 INIT_LIST_HEAD(&nfit_mem->list); 1062 nfit_mem->acpi_desc = acpi_desc; 1063 list_add(&nfit_mem->list, &acpi_desc->dimms); 1064 } 1065 1066 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) { 1067 if (nfit_dcr->dcr->region_index != dcr) 1068 continue; 1069 /* 1070 * Record the control region for the dimm. For 1071 * the ACPI 6.1 case, where there are separate 1072 * control regions for the pmem vs blk 1073 * interfaces, be sure to record the extended 1074 * blk details. 1075 */ 1076 if (!nfit_mem->dcr) 1077 nfit_mem->dcr = nfit_dcr->dcr; 1078 else if (nfit_mem->dcr->windows == 0 1079 && nfit_dcr->dcr->windows) 1080 nfit_mem->dcr = nfit_dcr->dcr; 1081 break; 1082 } 1083 1084 list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) { 1085 struct acpi_nfit_flush_address *flush; 1086 u16 i; 1087 1088 if (nfit_flush->flush->device_handle != device_handle) 1089 continue; 1090 nfit_mem->nfit_flush = nfit_flush; 1091 flush = nfit_flush->flush; 1092 nfit_mem->flush_wpq = devm_kzalloc(acpi_desc->dev, 1093 flush->hint_count 1094 * sizeof(struct resource), GFP_KERNEL); 1095 if (!nfit_mem->flush_wpq) 1096 return -ENOMEM; 1097 for (i = 0; i < flush->hint_count; i++) { 1098 struct resource *res = &nfit_mem->flush_wpq[i]; 1099 1100 res->start = flush->hint_address[i]; 1101 res->end = res->start + 8 - 1; 1102 } 1103 break; 1104 } 1105 1106 if (dcr && !nfit_mem->dcr) { 1107 dev_err(acpi_desc->dev, "SPA %d missing DCR %d\n", 1108 spa->range_index, dcr); 1109 return -ENODEV; 1110 } 1111 1112 if (type == NFIT_SPA_DCR) { 1113 struct nfit_idt *nfit_idt; 1114 u16 idt_idx; 1115 1116 /* multiple dimms may share a SPA when interleaved */ 1117 nfit_mem->spa_dcr = spa; 1118 nfit_mem->memdev_dcr = nfit_memdev->memdev; 1119 idt_idx = nfit_memdev->memdev->interleave_index; 1120 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) { 1121 if (nfit_idt->idt->interleave_index != idt_idx) 1122 continue; 1123 nfit_mem->idt_dcr = nfit_idt->idt; 1124 break; 1125 } 1126 nfit_mem_init_bdw(acpi_desc, nfit_mem, spa); 1127 } else if (type == NFIT_SPA_PM) { 1128 /* 1129 * A single dimm may belong to multiple SPA-PM 1130 * ranges, record at least one in addition to 1131 * any SPA-DCR range. 1132 */ 1133 nfit_mem->memdev_pmem = nfit_memdev->memdev; 1134 } else 1135 nfit_mem->memdev_dcr = nfit_memdev->memdev; 1136 } 1137 1138 return 0; 1139 } 1140 1141 static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b) 1142 { 1143 struct nfit_mem *a = container_of(_a, typeof(*a), list); 1144 struct nfit_mem *b = container_of(_b, typeof(*b), list); 1145 u32 handleA, handleB; 1146 1147 handleA = __to_nfit_memdev(a)->device_handle; 1148 handleB = __to_nfit_memdev(b)->device_handle; 1149 if (handleA < handleB) 1150 return -1; 1151 else if (handleA > handleB) 1152 return 1; 1153 return 0; 1154 } 1155 1156 static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc) 1157 { 1158 struct nfit_spa *nfit_spa; 1159 int rc; 1160 1161 1162 /* 1163 * For each SPA-DCR or SPA-PMEM address range find its 1164 * corresponding MEMDEV(s). From each MEMDEV find the 1165 * corresponding DCR. Then, if we're operating on a SPA-DCR, 1166 * try to find a SPA-BDW and a corresponding BDW that references 1167 * the DCR. Throw it all into an nfit_mem object. Note, that 1168 * BDWs are optional. 1169 */ 1170 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 1171 rc = __nfit_mem_init(acpi_desc, nfit_spa->spa); 1172 if (rc) 1173 return rc; 1174 } 1175 1176 /* 1177 * If a DIMM has failed to be mapped into SPA there will be no 1178 * SPA entries above. Find and register all the unmapped DIMMs 1179 * for reporting and recovery purposes. 1180 */ 1181 rc = __nfit_mem_init(acpi_desc, NULL); 1182 if (rc) 1183 return rc; 1184 1185 list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp); 1186 1187 return 0; 1188 } 1189 1190 static ssize_t bus_dsm_mask_show(struct device *dev, 1191 struct device_attribute *attr, char *buf) 1192 { 1193 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1194 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1195 1196 return sprintf(buf, "%#lx\n", nd_desc->bus_dsm_mask); 1197 } 1198 static struct device_attribute dev_attr_bus_dsm_mask = 1199 __ATTR(dsm_mask, 0444, bus_dsm_mask_show, NULL); 1200 1201 static ssize_t revision_show(struct device *dev, 1202 struct device_attribute *attr, char *buf) 1203 { 1204 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1205 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1206 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1207 1208 return sprintf(buf, "%d\n", acpi_desc->acpi_header.revision); 1209 } 1210 static DEVICE_ATTR_RO(revision); 1211 1212 static ssize_t hw_error_scrub_show(struct device *dev, 1213 struct device_attribute *attr, char *buf) 1214 { 1215 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1216 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1217 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1218 1219 return sprintf(buf, "%d\n", acpi_desc->scrub_mode); 1220 } 1221 1222 /* 1223 * The 'hw_error_scrub' attribute can have the following values written to it: 1224 * '0': Switch to the default mode where an exception will only insert 1225 * the address of the memory error into the poison and badblocks lists. 1226 * '1': Enable a full scrub to happen if an exception for a memory error is 1227 * received. 1228 */ 1229 static ssize_t hw_error_scrub_store(struct device *dev, 1230 struct device_attribute *attr, const char *buf, size_t size) 1231 { 1232 struct nvdimm_bus_descriptor *nd_desc; 1233 ssize_t rc; 1234 long val; 1235 1236 rc = kstrtol(buf, 0, &val); 1237 if (rc) 1238 return rc; 1239 1240 device_lock(dev); 1241 nd_desc = dev_get_drvdata(dev); 1242 if (nd_desc) { 1243 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1244 1245 switch (val) { 1246 case HW_ERROR_SCRUB_ON: 1247 acpi_desc->scrub_mode = HW_ERROR_SCRUB_ON; 1248 break; 1249 case HW_ERROR_SCRUB_OFF: 1250 acpi_desc->scrub_mode = HW_ERROR_SCRUB_OFF; 1251 break; 1252 default: 1253 rc = -EINVAL; 1254 break; 1255 } 1256 } 1257 device_unlock(dev); 1258 if (rc) 1259 return rc; 1260 return size; 1261 } 1262 static DEVICE_ATTR_RW(hw_error_scrub); 1263 1264 /* 1265 * This shows the number of full Address Range Scrubs that have been 1266 * completed since driver load time. Userspace can wait on this using 1267 * select/poll etc. A '+' at the end indicates an ARS is in progress 1268 */ 1269 static ssize_t scrub_show(struct device *dev, 1270 struct device_attribute *attr, char *buf) 1271 { 1272 struct nvdimm_bus_descriptor *nd_desc; 1273 ssize_t rc = -ENXIO; 1274 1275 device_lock(dev); 1276 nd_desc = dev_get_drvdata(dev); 1277 if (nd_desc) { 1278 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1279 1280 rc = sprintf(buf, "%d%s", acpi_desc->scrub_count, 1281 (work_busy(&acpi_desc->work)) ? "+\n" : "\n"); 1282 } 1283 device_unlock(dev); 1284 return rc; 1285 } 1286 1287 static ssize_t scrub_store(struct device *dev, 1288 struct device_attribute *attr, const char *buf, size_t size) 1289 { 1290 struct nvdimm_bus_descriptor *nd_desc; 1291 ssize_t rc; 1292 long val; 1293 1294 rc = kstrtol(buf, 0, &val); 1295 if (rc) 1296 return rc; 1297 if (val != 1) 1298 return -EINVAL; 1299 1300 device_lock(dev); 1301 nd_desc = dev_get_drvdata(dev); 1302 if (nd_desc) { 1303 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1304 1305 rc = acpi_nfit_ars_rescan(acpi_desc, 0); 1306 } 1307 device_unlock(dev); 1308 if (rc) 1309 return rc; 1310 return size; 1311 } 1312 static DEVICE_ATTR_RW(scrub); 1313 1314 static bool ars_supported(struct nvdimm_bus *nvdimm_bus) 1315 { 1316 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1317 const unsigned long mask = 1 << ND_CMD_ARS_CAP | 1 << ND_CMD_ARS_START 1318 | 1 << ND_CMD_ARS_STATUS; 1319 1320 return (nd_desc->cmd_mask & mask) == mask; 1321 } 1322 1323 static umode_t nfit_visible(struct kobject *kobj, struct attribute *a, int n) 1324 { 1325 struct device *dev = container_of(kobj, struct device, kobj); 1326 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1327 1328 if (a == &dev_attr_scrub.attr && !ars_supported(nvdimm_bus)) 1329 return 0; 1330 return a->mode; 1331 } 1332 1333 static struct attribute *acpi_nfit_attributes[] = { 1334 &dev_attr_revision.attr, 1335 &dev_attr_scrub.attr, 1336 &dev_attr_hw_error_scrub.attr, 1337 &dev_attr_bus_dsm_mask.attr, 1338 NULL, 1339 }; 1340 1341 static const struct attribute_group acpi_nfit_attribute_group = { 1342 .name = "nfit", 1343 .attrs = acpi_nfit_attributes, 1344 .is_visible = nfit_visible, 1345 }; 1346 1347 static const struct attribute_group *acpi_nfit_attribute_groups[] = { 1348 &nvdimm_bus_attribute_group, 1349 &acpi_nfit_attribute_group, 1350 NULL, 1351 }; 1352 1353 static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev) 1354 { 1355 struct nvdimm *nvdimm = to_nvdimm(dev); 1356 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1357 1358 return __to_nfit_memdev(nfit_mem); 1359 } 1360 1361 static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev) 1362 { 1363 struct nvdimm *nvdimm = to_nvdimm(dev); 1364 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1365 1366 return nfit_mem->dcr; 1367 } 1368 1369 static ssize_t handle_show(struct device *dev, 1370 struct device_attribute *attr, char *buf) 1371 { 1372 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev); 1373 1374 return sprintf(buf, "%#x\n", memdev->device_handle); 1375 } 1376 static DEVICE_ATTR_RO(handle); 1377 1378 static ssize_t phys_id_show(struct device *dev, 1379 struct device_attribute *attr, char *buf) 1380 { 1381 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev); 1382 1383 return sprintf(buf, "%#x\n", memdev->physical_id); 1384 } 1385 static DEVICE_ATTR_RO(phys_id); 1386 1387 static ssize_t vendor_show(struct device *dev, 1388 struct device_attribute *attr, char *buf) 1389 { 1390 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1391 1392 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->vendor_id)); 1393 } 1394 static DEVICE_ATTR_RO(vendor); 1395 1396 static ssize_t rev_id_show(struct device *dev, 1397 struct device_attribute *attr, char *buf) 1398 { 1399 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1400 1401 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->revision_id)); 1402 } 1403 static DEVICE_ATTR_RO(rev_id); 1404 1405 static ssize_t device_show(struct device *dev, 1406 struct device_attribute *attr, char *buf) 1407 { 1408 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1409 1410 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->device_id)); 1411 } 1412 static DEVICE_ATTR_RO(device); 1413 1414 static ssize_t subsystem_vendor_show(struct device *dev, 1415 struct device_attribute *attr, char *buf) 1416 { 1417 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1418 1419 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_vendor_id)); 1420 } 1421 static DEVICE_ATTR_RO(subsystem_vendor); 1422 1423 static ssize_t subsystem_rev_id_show(struct device *dev, 1424 struct device_attribute *attr, char *buf) 1425 { 1426 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1427 1428 return sprintf(buf, "0x%04x\n", 1429 be16_to_cpu(dcr->subsystem_revision_id)); 1430 } 1431 static DEVICE_ATTR_RO(subsystem_rev_id); 1432 1433 static ssize_t subsystem_device_show(struct device *dev, 1434 struct device_attribute *attr, char *buf) 1435 { 1436 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1437 1438 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_device_id)); 1439 } 1440 static DEVICE_ATTR_RO(subsystem_device); 1441 1442 static int num_nvdimm_formats(struct nvdimm *nvdimm) 1443 { 1444 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1445 int formats = 0; 1446 1447 if (nfit_mem->memdev_pmem) 1448 formats++; 1449 if (nfit_mem->memdev_bdw) 1450 formats++; 1451 return formats; 1452 } 1453 1454 static ssize_t format_show(struct device *dev, 1455 struct device_attribute *attr, char *buf) 1456 { 1457 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1458 1459 return sprintf(buf, "0x%04x\n", le16_to_cpu(dcr->code)); 1460 } 1461 static DEVICE_ATTR_RO(format); 1462 1463 static ssize_t format1_show(struct device *dev, 1464 struct device_attribute *attr, char *buf) 1465 { 1466 u32 handle; 1467 ssize_t rc = -ENXIO; 1468 struct nfit_mem *nfit_mem; 1469 struct nfit_memdev *nfit_memdev; 1470 struct acpi_nfit_desc *acpi_desc; 1471 struct nvdimm *nvdimm = to_nvdimm(dev); 1472 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1473 1474 nfit_mem = nvdimm_provider_data(nvdimm); 1475 acpi_desc = nfit_mem->acpi_desc; 1476 handle = to_nfit_memdev(dev)->device_handle; 1477 1478 /* assumes DIMMs have at most 2 published interface codes */ 1479 mutex_lock(&acpi_desc->init_mutex); 1480 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 1481 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev; 1482 struct nfit_dcr *nfit_dcr; 1483 1484 if (memdev->device_handle != handle) 1485 continue; 1486 1487 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) { 1488 if (nfit_dcr->dcr->region_index != memdev->region_index) 1489 continue; 1490 if (nfit_dcr->dcr->code == dcr->code) 1491 continue; 1492 rc = sprintf(buf, "0x%04x\n", 1493 le16_to_cpu(nfit_dcr->dcr->code)); 1494 break; 1495 } 1496 if (rc != ENXIO) 1497 break; 1498 } 1499 mutex_unlock(&acpi_desc->init_mutex); 1500 return rc; 1501 } 1502 static DEVICE_ATTR_RO(format1); 1503 1504 static ssize_t formats_show(struct device *dev, 1505 struct device_attribute *attr, char *buf) 1506 { 1507 struct nvdimm *nvdimm = to_nvdimm(dev); 1508 1509 return sprintf(buf, "%d\n", num_nvdimm_formats(nvdimm)); 1510 } 1511 static DEVICE_ATTR_RO(formats); 1512 1513 static ssize_t serial_show(struct device *dev, 1514 struct device_attribute *attr, char *buf) 1515 { 1516 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1517 1518 return sprintf(buf, "0x%08x\n", be32_to_cpu(dcr->serial_number)); 1519 } 1520 static DEVICE_ATTR_RO(serial); 1521 1522 static ssize_t family_show(struct device *dev, 1523 struct device_attribute *attr, char *buf) 1524 { 1525 struct nvdimm *nvdimm = to_nvdimm(dev); 1526 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1527 1528 if (nfit_mem->family < 0) 1529 return -ENXIO; 1530 return sprintf(buf, "%d\n", nfit_mem->family); 1531 } 1532 static DEVICE_ATTR_RO(family); 1533 1534 static ssize_t dsm_mask_show(struct device *dev, 1535 struct device_attribute *attr, char *buf) 1536 { 1537 struct nvdimm *nvdimm = to_nvdimm(dev); 1538 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1539 1540 if (nfit_mem->family < 0) 1541 return -ENXIO; 1542 return sprintf(buf, "%#lx\n", nfit_mem->dsm_mask); 1543 } 1544 static DEVICE_ATTR_RO(dsm_mask); 1545 1546 static ssize_t flags_show(struct device *dev, 1547 struct device_attribute *attr, char *buf) 1548 { 1549 u16 flags = to_nfit_memdev(dev)->flags; 1550 1551 return sprintf(buf, "%s%s%s%s%s%s%s\n", 1552 flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "", 1553 flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "", 1554 flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "", 1555 flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "", 1556 flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : "", 1557 flags & ACPI_NFIT_MEM_MAP_FAILED ? "map_fail " : "", 1558 flags & ACPI_NFIT_MEM_HEALTH_ENABLED ? "smart_notify " : ""); 1559 } 1560 static DEVICE_ATTR_RO(flags); 1561 1562 static ssize_t id_show(struct device *dev, 1563 struct device_attribute *attr, char *buf) 1564 { 1565 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1566 1567 if (dcr->valid_fields & ACPI_NFIT_CONTROL_MFG_INFO_VALID) 1568 return sprintf(buf, "%04x-%02x-%04x-%08x\n", 1569 be16_to_cpu(dcr->vendor_id), 1570 dcr->manufacturing_location, 1571 be16_to_cpu(dcr->manufacturing_date), 1572 be32_to_cpu(dcr->serial_number)); 1573 else 1574 return sprintf(buf, "%04x-%08x\n", 1575 be16_to_cpu(dcr->vendor_id), 1576 be32_to_cpu(dcr->serial_number)); 1577 } 1578 static DEVICE_ATTR_RO(id); 1579 1580 static struct attribute *acpi_nfit_dimm_attributes[] = { 1581 &dev_attr_handle.attr, 1582 &dev_attr_phys_id.attr, 1583 &dev_attr_vendor.attr, 1584 &dev_attr_device.attr, 1585 &dev_attr_rev_id.attr, 1586 &dev_attr_subsystem_vendor.attr, 1587 &dev_attr_subsystem_device.attr, 1588 &dev_attr_subsystem_rev_id.attr, 1589 &dev_attr_format.attr, 1590 &dev_attr_formats.attr, 1591 &dev_attr_format1.attr, 1592 &dev_attr_serial.attr, 1593 &dev_attr_flags.attr, 1594 &dev_attr_id.attr, 1595 &dev_attr_family.attr, 1596 &dev_attr_dsm_mask.attr, 1597 NULL, 1598 }; 1599 1600 static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj, 1601 struct attribute *a, int n) 1602 { 1603 struct device *dev = container_of(kobj, struct device, kobj); 1604 struct nvdimm *nvdimm = to_nvdimm(dev); 1605 1606 if (!to_nfit_dcr(dev)) { 1607 /* Without a dcr only the memdev attributes can be surfaced */ 1608 if (a == &dev_attr_handle.attr || a == &dev_attr_phys_id.attr 1609 || a == &dev_attr_flags.attr 1610 || a == &dev_attr_family.attr 1611 || a == &dev_attr_dsm_mask.attr) 1612 return a->mode; 1613 return 0; 1614 } 1615 1616 if (a == &dev_attr_format1.attr && num_nvdimm_formats(nvdimm) <= 1) 1617 return 0; 1618 return a->mode; 1619 } 1620 1621 static const struct attribute_group acpi_nfit_dimm_attribute_group = { 1622 .name = "nfit", 1623 .attrs = acpi_nfit_dimm_attributes, 1624 .is_visible = acpi_nfit_dimm_attr_visible, 1625 }; 1626 1627 static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = { 1628 &nvdimm_attribute_group, 1629 &nd_device_attribute_group, 1630 &acpi_nfit_dimm_attribute_group, 1631 NULL, 1632 }; 1633 1634 static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc, 1635 u32 device_handle) 1636 { 1637 struct nfit_mem *nfit_mem; 1638 1639 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) 1640 if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle) 1641 return nfit_mem->nvdimm; 1642 1643 return NULL; 1644 } 1645 1646 void __acpi_nvdimm_notify(struct device *dev, u32 event) 1647 { 1648 struct nfit_mem *nfit_mem; 1649 struct acpi_nfit_desc *acpi_desc; 1650 1651 dev_dbg(dev->parent, "%s: %s: event: %d\n", dev_name(dev), __func__, 1652 event); 1653 1654 if (event != NFIT_NOTIFY_DIMM_HEALTH) { 1655 dev_dbg(dev->parent, "%s: unknown event: %d\n", dev_name(dev), 1656 event); 1657 return; 1658 } 1659 1660 acpi_desc = dev_get_drvdata(dev->parent); 1661 if (!acpi_desc) 1662 return; 1663 1664 /* 1665 * If we successfully retrieved acpi_desc, then we know nfit_mem data 1666 * is still valid. 1667 */ 1668 nfit_mem = dev_get_drvdata(dev); 1669 if (nfit_mem && nfit_mem->flags_attr) 1670 sysfs_notify_dirent(nfit_mem->flags_attr); 1671 } 1672 EXPORT_SYMBOL_GPL(__acpi_nvdimm_notify); 1673 1674 static void acpi_nvdimm_notify(acpi_handle handle, u32 event, void *data) 1675 { 1676 struct acpi_device *adev = data; 1677 struct device *dev = &adev->dev; 1678 1679 device_lock(dev->parent); 1680 __acpi_nvdimm_notify(dev, event); 1681 device_unlock(dev->parent); 1682 } 1683 1684 static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc, 1685 struct nfit_mem *nfit_mem, u32 device_handle) 1686 { 1687 struct acpi_device *adev, *adev_dimm; 1688 struct device *dev = acpi_desc->dev; 1689 union acpi_object *obj; 1690 unsigned long dsm_mask; 1691 const guid_t *guid; 1692 int i; 1693 int family = -1; 1694 1695 /* nfit test assumes 1:1 relationship between commands and dsms */ 1696 nfit_mem->dsm_mask = acpi_desc->dimm_cmd_force_en; 1697 nfit_mem->family = NVDIMM_FAMILY_INTEL; 1698 adev = to_acpi_dev(acpi_desc); 1699 if (!adev) 1700 return 0; 1701 1702 adev_dimm = acpi_find_child_device(adev, device_handle, false); 1703 nfit_mem->adev = adev_dimm; 1704 if (!adev_dimm) { 1705 dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n", 1706 device_handle); 1707 return force_enable_dimms ? 0 : -ENODEV; 1708 } 1709 1710 if (ACPI_FAILURE(acpi_install_notify_handler(adev_dimm->handle, 1711 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify, adev_dimm))) { 1712 dev_err(dev, "%s: notification registration failed\n", 1713 dev_name(&adev_dimm->dev)); 1714 return -ENXIO; 1715 } 1716 /* 1717 * Record nfit_mem for the notification path to track back to 1718 * the nfit sysfs attributes for this dimm device object. 1719 */ 1720 dev_set_drvdata(&adev_dimm->dev, nfit_mem); 1721 1722 /* 1723 * Until standardization materializes we need to consider 4 1724 * different command sets. Note, that checking for function0 (bit0) 1725 * tells us if any commands are reachable through this GUID. 1726 */ 1727 for (i = 0; i <= NVDIMM_FAMILY_MAX; i++) 1728 if (acpi_check_dsm(adev_dimm->handle, to_nfit_uuid(i), 1, 1)) 1729 if (family < 0 || i == default_dsm_family) 1730 family = i; 1731 1732 /* limit the supported commands to those that are publicly documented */ 1733 nfit_mem->family = family; 1734 if (override_dsm_mask && !disable_vendor_specific) 1735 dsm_mask = override_dsm_mask; 1736 else if (nfit_mem->family == NVDIMM_FAMILY_INTEL) { 1737 dsm_mask = NVDIMM_INTEL_CMDMASK; 1738 if (disable_vendor_specific) 1739 dsm_mask &= ~(1 << ND_CMD_VENDOR); 1740 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE1) { 1741 dsm_mask = 0x1c3c76; 1742 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE2) { 1743 dsm_mask = 0x1fe; 1744 if (disable_vendor_specific) 1745 dsm_mask &= ~(1 << 8); 1746 } else if (nfit_mem->family == NVDIMM_FAMILY_MSFT) { 1747 dsm_mask = 0xffffffff; 1748 } else { 1749 dev_dbg(dev, "unknown dimm command family\n"); 1750 nfit_mem->family = -1; 1751 /* DSMs are optional, continue loading the driver... */ 1752 return 0; 1753 } 1754 1755 guid = to_nfit_uuid(nfit_mem->family); 1756 for_each_set_bit(i, &dsm_mask, BITS_PER_LONG) 1757 if (acpi_check_dsm(adev_dimm->handle, guid, 1758 nfit_dsm_revid(nfit_mem->family, i), 1759 1ULL << i)) 1760 set_bit(i, &nfit_mem->dsm_mask); 1761 1762 obj = acpi_label_info(adev_dimm->handle); 1763 if (obj) { 1764 ACPI_FREE(obj); 1765 nfit_mem->has_lsi = 1; 1766 dev_dbg(dev, "%s: has _LSI\n", dev_name(&adev_dimm->dev)); 1767 } 1768 1769 obj = acpi_label_read(adev_dimm->handle, 0, 0); 1770 if (obj) { 1771 ACPI_FREE(obj); 1772 nfit_mem->has_lsr = 1; 1773 dev_dbg(dev, "%s: has _LSR\n", dev_name(&adev_dimm->dev)); 1774 } 1775 1776 obj = acpi_label_write(adev_dimm->handle, 0, 0, NULL); 1777 if (obj) { 1778 ACPI_FREE(obj); 1779 nfit_mem->has_lsw = 1; 1780 dev_dbg(dev, "%s: has _LSW\n", dev_name(&adev_dimm->dev)); 1781 } 1782 1783 return 0; 1784 } 1785 1786 static void shutdown_dimm_notify(void *data) 1787 { 1788 struct acpi_nfit_desc *acpi_desc = data; 1789 struct nfit_mem *nfit_mem; 1790 1791 mutex_lock(&acpi_desc->init_mutex); 1792 /* 1793 * Clear out the nfit_mem->flags_attr and shut down dimm event 1794 * notifications. 1795 */ 1796 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1797 struct acpi_device *adev_dimm = nfit_mem->adev; 1798 1799 if (nfit_mem->flags_attr) { 1800 sysfs_put(nfit_mem->flags_attr); 1801 nfit_mem->flags_attr = NULL; 1802 } 1803 if (adev_dimm) { 1804 acpi_remove_notify_handler(adev_dimm->handle, 1805 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify); 1806 dev_set_drvdata(&adev_dimm->dev, NULL); 1807 } 1808 } 1809 mutex_unlock(&acpi_desc->init_mutex); 1810 } 1811 1812 static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc) 1813 { 1814 struct nfit_mem *nfit_mem; 1815 int dimm_count = 0, rc; 1816 struct nvdimm *nvdimm; 1817 1818 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1819 struct acpi_nfit_flush_address *flush; 1820 unsigned long flags = 0, cmd_mask; 1821 struct nfit_memdev *nfit_memdev; 1822 u32 device_handle; 1823 u16 mem_flags; 1824 1825 device_handle = __to_nfit_memdev(nfit_mem)->device_handle; 1826 nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle); 1827 if (nvdimm) { 1828 dimm_count++; 1829 continue; 1830 } 1831 1832 if (nfit_mem->bdw && nfit_mem->memdev_pmem) 1833 set_bit(NDD_ALIASING, &flags); 1834 1835 /* collate flags across all memdevs for this dimm */ 1836 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 1837 struct acpi_nfit_memory_map *dimm_memdev; 1838 1839 dimm_memdev = __to_nfit_memdev(nfit_mem); 1840 if (dimm_memdev->device_handle 1841 != nfit_memdev->memdev->device_handle) 1842 continue; 1843 dimm_memdev->flags |= nfit_memdev->memdev->flags; 1844 } 1845 1846 mem_flags = __to_nfit_memdev(nfit_mem)->flags; 1847 if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED) 1848 set_bit(NDD_UNARMED, &flags); 1849 1850 rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle); 1851 if (rc) 1852 continue; 1853 1854 /* 1855 * TODO: provide translation for non-NVDIMM_FAMILY_INTEL 1856 * devices (i.e. from nd_cmd to acpi_dsm) to standardize the 1857 * userspace interface. 1858 */ 1859 cmd_mask = 1UL << ND_CMD_CALL; 1860 if (nfit_mem->family == NVDIMM_FAMILY_INTEL) { 1861 /* 1862 * These commands have a 1:1 correspondence 1863 * between DSM payload and libnvdimm ioctl 1864 * payload format. 1865 */ 1866 cmd_mask |= nfit_mem->dsm_mask & NVDIMM_STANDARD_CMDMASK; 1867 } 1868 1869 if (nfit_mem->has_lsi) 1870 set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask); 1871 if (nfit_mem->has_lsr) 1872 set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask); 1873 if (nfit_mem->has_lsw) 1874 set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask); 1875 1876 flush = nfit_mem->nfit_flush ? nfit_mem->nfit_flush->flush 1877 : NULL; 1878 nvdimm = nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem, 1879 acpi_nfit_dimm_attribute_groups, 1880 flags, cmd_mask, flush ? flush->hint_count : 0, 1881 nfit_mem->flush_wpq); 1882 if (!nvdimm) 1883 return -ENOMEM; 1884 1885 nfit_mem->nvdimm = nvdimm; 1886 dimm_count++; 1887 1888 if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0) 1889 continue; 1890 1891 dev_info(acpi_desc->dev, "%s flags:%s%s%s%s%s\n", 1892 nvdimm_name(nvdimm), 1893 mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "", 1894 mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"", 1895 mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "", 1896 mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : "", 1897 mem_flags & ACPI_NFIT_MEM_MAP_FAILED ? " map_fail" : ""); 1898 1899 } 1900 1901 rc = nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count); 1902 if (rc) 1903 return rc; 1904 1905 /* 1906 * Now that dimms are successfully registered, and async registration 1907 * is flushed, attempt to enable event notification. 1908 */ 1909 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1910 struct kernfs_node *nfit_kernfs; 1911 1912 nvdimm = nfit_mem->nvdimm; 1913 if (!nvdimm) 1914 continue; 1915 1916 nfit_kernfs = sysfs_get_dirent(nvdimm_kobj(nvdimm)->sd, "nfit"); 1917 if (nfit_kernfs) 1918 nfit_mem->flags_attr = sysfs_get_dirent(nfit_kernfs, 1919 "flags"); 1920 sysfs_put(nfit_kernfs); 1921 if (!nfit_mem->flags_attr) 1922 dev_warn(acpi_desc->dev, "%s: notifications disabled\n", 1923 nvdimm_name(nvdimm)); 1924 } 1925 1926 return devm_add_action_or_reset(acpi_desc->dev, shutdown_dimm_notify, 1927 acpi_desc); 1928 } 1929 1930 /* 1931 * These constants are private because there are no kernel consumers of 1932 * these commands. 1933 */ 1934 enum nfit_aux_cmds { 1935 NFIT_CMD_TRANSLATE_SPA = 5, 1936 NFIT_CMD_ARS_INJECT_SET = 7, 1937 NFIT_CMD_ARS_INJECT_CLEAR = 8, 1938 NFIT_CMD_ARS_INJECT_GET = 9, 1939 }; 1940 1941 static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc) 1942 { 1943 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 1944 const guid_t *guid = to_nfit_uuid(NFIT_DEV_BUS); 1945 struct acpi_device *adev; 1946 unsigned long dsm_mask; 1947 int i; 1948 1949 nd_desc->cmd_mask = acpi_desc->bus_cmd_force_en; 1950 nd_desc->bus_dsm_mask = acpi_desc->bus_nfit_cmd_force_en; 1951 adev = to_acpi_dev(acpi_desc); 1952 if (!adev) 1953 return; 1954 1955 for (i = ND_CMD_ARS_CAP; i <= ND_CMD_CLEAR_ERROR; i++) 1956 if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i)) 1957 set_bit(i, &nd_desc->cmd_mask); 1958 set_bit(ND_CMD_CALL, &nd_desc->cmd_mask); 1959 1960 dsm_mask = 1961 (1 << ND_CMD_ARS_CAP) | 1962 (1 << ND_CMD_ARS_START) | 1963 (1 << ND_CMD_ARS_STATUS) | 1964 (1 << ND_CMD_CLEAR_ERROR) | 1965 (1 << NFIT_CMD_TRANSLATE_SPA) | 1966 (1 << NFIT_CMD_ARS_INJECT_SET) | 1967 (1 << NFIT_CMD_ARS_INJECT_CLEAR) | 1968 (1 << NFIT_CMD_ARS_INJECT_GET); 1969 for_each_set_bit(i, &dsm_mask, BITS_PER_LONG) 1970 if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i)) 1971 set_bit(i, &nd_desc->bus_dsm_mask); 1972 } 1973 1974 static ssize_t range_index_show(struct device *dev, 1975 struct device_attribute *attr, char *buf) 1976 { 1977 struct nd_region *nd_region = to_nd_region(dev); 1978 struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region); 1979 1980 return sprintf(buf, "%d\n", nfit_spa->spa->range_index); 1981 } 1982 static DEVICE_ATTR_RO(range_index); 1983 1984 static ssize_t ecc_unit_size_show(struct device *dev, 1985 struct device_attribute *attr, char *buf) 1986 { 1987 struct nd_region *nd_region = to_nd_region(dev); 1988 struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region); 1989 1990 return sprintf(buf, "%d\n", nfit_spa->clear_err_unit); 1991 } 1992 static DEVICE_ATTR_RO(ecc_unit_size); 1993 1994 static struct attribute *acpi_nfit_region_attributes[] = { 1995 &dev_attr_range_index.attr, 1996 &dev_attr_ecc_unit_size.attr, 1997 NULL, 1998 }; 1999 2000 static const struct attribute_group acpi_nfit_region_attribute_group = { 2001 .name = "nfit", 2002 .attrs = acpi_nfit_region_attributes, 2003 }; 2004 2005 static const struct attribute_group *acpi_nfit_region_attribute_groups[] = { 2006 &nd_region_attribute_group, 2007 &nd_mapping_attribute_group, 2008 &nd_device_attribute_group, 2009 &nd_numa_attribute_group, 2010 &acpi_nfit_region_attribute_group, 2011 NULL, 2012 }; 2013 2014 /* enough info to uniquely specify an interleave set */ 2015 struct nfit_set_info { 2016 struct nfit_set_info_map { 2017 u64 region_offset; 2018 u32 serial_number; 2019 u32 pad; 2020 } mapping[0]; 2021 }; 2022 2023 struct nfit_set_info2 { 2024 struct nfit_set_info_map2 { 2025 u64 region_offset; 2026 u32 serial_number; 2027 u16 vendor_id; 2028 u16 manufacturing_date; 2029 u8 manufacturing_location; 2030 u8 reserved[31]; 2031 } mapping[0]; 2032 }; 2033 2034 static size_t sizeof_nfit_set_info(int num_mappings) 2035 { 2036 return sizeof(struct nfit_set_info) 2037 + num_mappings * sizeof(struct nfit_set_info_map); 2038 } 2039 2040 static size_t sizeof_nfit_set_info2(int num_mappings) 2041 { 2042 return sizeof(struct nfit_set_info2) 2043 + num_mappings * sizeof(struct nfit_set_info_map2); 2044 } 2045 2046 static int cmp_map_compat(const void *m0, const void *m1) 2047 { 2048 const struct nfit_set_info_map *map0 = m0; 2049 const struct nfit_set_info_map *map1 = m1; 2050 2051 return memcmp(&map0->region_offset, &map1->region_offset, 2052 sizeof(u64)); 2053 } 2054 2055 static int cmp_map(const void *m0, const void *m1) 2056 { 2057 const struct nfit_set_info_map *map0 = m0; 2058 const struct nfit_set_info_map *map1 = m1; 2059 2060 if (map0->region_offset < map1->region_offset) 2061 return -1; 2062 else if (map0->region_offset > map1->region_offset) 2063 return 1; 2064 return 0; 2065 } 2066 2067 static int cmp_map2(const void *m0, const void *m1) 2068 { 2069 const struct nfit_set_info_map2 *map0 = m0; 2070 const struct nfit_set_info_map2 *map1 = m1; 2071 2072 if (map0->region_offset < map1->region_offset) 2073 return -1; 2074 else if (map0->region_offset > map1->region_offset) 2075 return 1; 2076 return 0; 2077 } 2078 2079 /* Retrieve the nth entry referencing this spa */ 2080 static struct acpi_nfit_memory_map *memdev_from_spa( 2081 struct acpi_nfit_desc *acpi_desc, u16 range_index, int n) 2082 { 2083 struct nfit_memdev *nfit_memdev; 2084 2085 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) 2086 if (nfit_memdev->memdev->range_index == range_index) 2087 if (n-- == 0) 2088 return nfit_memdev->memdev; 2089 return NULL; 2090 } 2091 2092 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc, 2093 struct nd_region_desc *ndr_desc, 2094 struct acpi_nfit_system_address *spa) 2095 { 2096 struct device *dev = acpi_desc->dev; 2097 struct nd_interleave_set *nd_set; 2098 u16 nr = ndr_desc->num_mappings; 2099 struct nfit_set_info2 *info2; 2100 struct nfit_set_info *info; 2101 int i; 2102 2103 nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL); 2104 if (!nd_set) 2105 return -ENOMEM; 2106 ndr_desc->nd_set = nd_set; 2107 guid_copy(&nd_set->type_guid, (guid_t *) spa->range_guid); 2108 2109 info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL); 2110 if (!info) 2111 return -ENOMEM; 2112 2113 info2 = devm_kzalloc(dev, sizeof_nfit_set_info2(nr), GFP_KERNEL); 2114 if (!info2) 2115 return -ENOMEM; 2116 2117 for (i = 0; i < nr; i++) { 2118 struct nd_mapping_desc *mapping = &ndr_desc->mapping[i]; 2119 struct nfit_set_info_map *map = &info->mapping[i]; 2120 struct nfit_set_info_map2 *map2 = &info2->mapping[i]; 2121 struct nvdimm *nvdimm = mapping->nvdimm; 2122 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 2123 struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc, 2124 spa->range_index, i); 2125 struct acpi_nfit_control_region *dcr = nfit_mem->dcr; 2126 2127 if (!memdev || !nfit_mem->dcr) { 2128 dev_err(dev, "%s: failed to find DCR\n", __func__); 2129 return -ENODEV; 2130 } 2131 2132 map->region_offset = memdev->region_offset; 2133 map->serial_number = dcr->serial_number; 2134 2135 map2->region_offset = memdev->region_offset; 2136 map2->serial_number = dcr->serial_number; 2137 map2->vendor_id = dcr->vendor_id; 2138 map2->manufacturing_date = dcr->manufacturing_date; 2139 map2->manufacturing_location = dcr->manufacturing_location; 2140 } 2141 2142 /* v1.1 namespaces */ 2143 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map), 2144 cmp_map, NULL); 2145 nd_set->cookie1 = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0); 2146 2147 /* v1.2 namespaces */ 2148 sort(&info2->mapping[0], nr, sizeof(struct nfit_set_info_map2), 2149 cmp_map2, NULL); 2150 nd_set->cookie2 = nd_fletcher64(info2, sizeof_nfit_set_info2(nr), 0); 2151 2152 /* support v1.1 namespaces created with the wrong sort order */ 2153 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map), 2154 cmp_map_compat, NULL); 2155 nd_set->altcookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0); 2156 2157 /* record the result of the sort for the mapping position */ 2158 for (i = 0; i < nr; i++) { 2159 struct nfit_set_info_map2 *map2 = &info2->mapping[i]; 2160 int j; 2161 2162 for (j = 0; j < nr; j++) { 2163 struct nd_mapping_desc *mapping = &ndr_desc->mapping[j]; 2164 struct nvdimm *nvdimm = mapping->nvdimm; 2165 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 2166 struct acpi_nfit_control_region *dcr = nfit_mem->dcr; 2167 2168 if (map2->serial_number == dcr->serial_number && 2169 map2->vendor_id == dcr->vendor_id && 2170 map2->manufacturing_date == dcr->manufacturing_date && 2171 map2->manufacturing_location 2172 == dcr->manufacturing_location) { 2173 mapping->position = i; 2174 break; 2175 } 2176 } 2177 } 2178 2179 ndr_desc->nd_set = nd_set; 2180 devm_kfree(dev, info); 2181 devm_kfree(dev, info2); 2182 2183 return 0; 2184 } 2185 2186 static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio) 2187 { 2188 struct acpi_nfit_interleave *idt = mmio->idt; 2189 u32 sub_line_offset, line_index, line_offset; 2190 u64 line_no, table_skip_count, table_offset; 2191 2192 line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset); 2193 table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index); 2194 line_offset = idt->line_offset[line_index] 2195 * mmio->line_size; 2196 table_offset = table_skip_count * mmio->table_size; 2197 2198 return mmio->base_offset + line_offset + table_offset + sub_line_offset; 2199 } 2200 2201 static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw) 2202 { 2203 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; 2204 u64 offset = nfit_blk->stat_offset + mmio->size * bw; 2205 const u32 STATUS_MASK = 0x80000037; 2206 2207 if (mmio->num_lines) 2208 offset = to_interleave_offset(offset, mmio); 2209 2210 return readl(mmio->addr.base + offset) & STATUS_MASK; 2211 } 2212 2213 static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw, 2214 resource_size_t dpa, unsigned int len, unsigned int write) 2215 { 2216 u64 cmd, offset; 2217 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; 2218 2219 enum { 2220 BCW_OFFSET_MASK = (1ULL << 48)-1, 2221 BCW_LEN_SHIFT = 48, 2222 BCW_LEN_MASK = (1ULL << 8) - 1, 2223 BCW_CMD_SHIFT = 56, 2224 }; 2225 2226 cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK; 2227 len = len >> L1_CACHE_SHIFT; 2228 cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT; 2229 cmd |= ((u64) write) << BCW_CMD_SHIFT; 2230 2231 offset = nfit_blk->cmd_offset + mmio->size * bw; 2232 if (mmio->num_lines) 2233 offset = to_interleave_offset(offset, mmio); 2234 2235 writeq(cmd, mmio->addr.base + offset); 2236 nvdimm_flush(nfit_blk->nd_region); 2237 2238 if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH) 2239 readq(mmio->addr.base + offset); 2240 } 2241 2242 static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk, 2243 resource_size_t dpa, void *iobuf, size_t len, int rw, 2244 unsigned int lane) 2245 { 2246 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW]; 2247 unsigned int copied = 0; 2248 u64 base_offset; 2249 int rc; 2250 2251 base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES 2252 + lane * mmio->size; 2253 write_blk_ctl(nfit_blk, lane, dpa, len, rw); 2254 while (len) { 2255 unsigned int c; 2256 u64 offset; 2257 2258 if (mmio->num_lines) { 2259 u32 line_offset; 2260 2261 offset = to_interleave_offset(base_offset + copied, 2262 mmio); 2263 div_u64_rem(offset, mmio->line_size, &line_offset); 2264 c = min_t(size_t, len, mmio->line_size - line_offset); 2265 } else { 2266 offset = base_offset + nfit_blk->bdw_offset; 2267 c = len; 2268 } 2269 2270 if (rw) 2271 memcpy_flushcache(mmio->addr.aperture + offset, iobuf + copied, c); 2272 else { 2273 if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH) 2274 arch_invalidate_pmem((void __force *) 2275 mmio->addr.aperture + offset, c); 2276 2277 memcpy(iobuf + copied, mmio->addr.aperture + offset, c); 2278 } 2279 2280 copied += c; 2281 len -= c; 2282 } 2283 2284 if (rw) 2285 nvdimm_flush(nfit_blk->nd_region); 2286 2287 rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0; 2288 return rc; 2289 } 2290 2291 static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr, 2292 resource_size_t dpa, void *iobuf, u64 len, int rw) 2293 { 2294 struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr); 2295 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW]; 2296 struct nd_region *nd_region = nfit_blk->nd_region; 2297 unsigned int lane, copied = 0; 2298 int rc = 0; 2299 2300 lane = nd_region_acquire_lane(nd_region); 2301 while (len) { 2302 u64 c = min(len, mmio->size); 2303 2304 rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied, 2305 iobuf + copied, c, rw, lane); 2306 if (rc) 2307 break; 2308 2309 copied += c; 2310 len -= c; 2311 } 2312 nd_region_release_lane(nd_region, lane); 2313 2314 return rc; 2315 } 2316 2317 static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio, 2318 struct acpi_nfit_interleave *idt, u16 interleave_ways) 2319 { 2320 if (idt) { 2321 mmio->num_lines = idt->line_count; 2322 mmio->line_size = idt->line_size; 2323 if (interleave_ways == 0) 2324 return -ENXIO; 2325 mmio->table_size = mmio->num_lines * interleave_ways 2326 * mmio->line_size; 2327 } 2328 2329 return 0; 2330 } 2331 2332 static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc, 2333 struct nvdimm *nvdimm, struct nfit_blk *nfit_blk) 2334 { 2335 struct nd_cmd_dimm_flags flags; 2336 int rc; 2337 2338 memset(&flags, 0, sizeof(flags)); 2339 rc = nd_desc->ndctl(nd_desc, nvdimm, ND_CMD_DIMM_FLAGS, &flags, 2340 sizeof(flags), NULL); 2341 2342 if (rc >= 0 && flags.status == 0) 2343 nfit_blk->dimm_flags = flags.flags; 2344 else if (rc == -ENOTTY) { 2345 /* fall back to a conservative default */ 2346 nfit_blk->dimm_flags = NFIT_BLK_DCR_LATCH | NFIT_BLK_READ_FLUSH; 2347 rc = 0; 2348 } else 2349 rc = -ENXIO; 2350 2351 return rc; 2352 } 2353 2354 static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, 2355 struct device *dev) 2356 { 2357 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 2358 struct nd_blk_region *ndbr = to_nd_blk_region(dev); 2359 struct nfit_blk_mmio *mmio; 2360 struct nfit_blk *nfit_blk; 2361 struct nfit_mem *nfit_mem; 2362 struct nvdimm *nvdimm; 2363 int rc; 2364 2365 nvdimm = nd_blk_region_to_dimm(ndbr); 2366 nfit_mem = nvdimm_provider_data(nvdimm); 2367 if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) { 2368 dev_dbg(dev, "%s: missing%s%s%s\n", __func__, 2369 nfit_mem ? "" : " nfit_mem", 2370 (nfit_mem && nfit_mem->dcr) ? "" : " dcr", 2371 (nfit_mem && nfit_mem->bdw) ? "" : " bdw"); 2372 return -ENXIO; 2373 } 2374 2375 nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL); 2376 if (!nfit_blk) 2377 return -ENOMEM; 2378 nd_blk_region_set_provider_data(ndbr, nfit_blk); 2379 nfit_blk->nd_region = to_nd_region(dev); 2380 2381 /* map block aperture memory */ 2382 nfit_blk->bdw_offset = nfit_mem->bdw->offset; 2383 mmio = &nfit_blk->mmio[BDW]; 2384 mmio->addr.base = devm_nvdimm_memremap(dev, nfit_mem->spa_bdw->address, 2385 nfit_mem->spa_bdw->length, nd_blk_memremap_flags(ndbr)); 2386 if (!mmio->addr.base) { 2387 dev_dbg(dev, "%s: %s failed to map bdw\n", __func__, 2388 nvdimm_name(nvdimm)); 2389 return -ENOMEM; 2390 } 2391 mmio->size = nfit_mem->bdw->size; 2392 mmio->base_offset = nfit_mem->memdev_bdw->region_offset; 2393 mmio->idt = nfit_mem->idt_bdw; 2394 mmio->spa = nfit_mem->spa_bdw; 2395 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw, 2396 nfit_mem->memdev_bdw->interleave_ways); 2397 if (rc) { 2398 dev_dbg(dev, "%s: %s failed to init bdw interleave\n", 2399 __func__, nvdimm_name(nvdimm)); 2400 return rc; 2401 } 2402 2403 /* map block control memory */ 2404 nfit_blk->cmd_offset = nfit_mem->dcr->command_offset; 2405 nfit_blk->stat_offset = nfit_mem->dcr->status_offset; 2406 mmio = &nfit_blk->mmio[DCR]; 2407 mmio->addr.base = devm_nvdimm_ioremap(dev, nfit_mem->spa_dcr->address, 2408 nfit_mem->spa_dcr->length); 2409 if (!mmio->addr.base) { 2410 dev_dbg(dev, "%s: %s failed to map dcr\n", __func__, 2411 nvdimm_name(nvdimm)); 2412 return -ENOMEM; 2413 } 2414 mmio->size = nfit_mem->dcr->window_size; 2415 mmio->base_offset = nfit_mem->memdev_dcr->region_offset; 2416 mmio->idt = nfit_mem->idt_dcr; 2417 mmio->spa = nfit_mem->spa_dcr; 2418 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr, 2419 nfit_mem->memdev_dcr->interleave_ways); 2420 if (rc) { 2421 dev_dbg(dev, "%s: %s failed to init dcr interleave\n", 2422 __func__, nvdimm_name(nvdimm)); 2423 return rc; 2424 } 2425 2426 rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk); 2427 if (rc < 0) { 2428 dev_dbg(dev, "%s: %s failed get DIMM flags\n", 2429 __func__, nvdimm_name(nvdimm)); 2430 return rc; 2431 } 2432 2433 if (nvdimm_has_flush(nfit_blk->nd_region) < 0) 2434 dev_warn(dev, "unable to guarantee persistence of writes\n"); 2435 2436 if (mmio->line_size == 0) 2437 return 0; 2438 2439 if ((u32) nfit_blk->cmd_offset % mmio->line_size 2440 + 8 > mmio->line_size) { 2441 dev_dbg(dev, "cmd_offset crosses interleave boundary\n"); 2442 return -ENXIO; 2443 } else if ((u32) nfit_blk->stat_offset % mmio->line_size 2444 + 8 > mmio->line_size) { 2445 dev_dbg(dev, "stat_offset crosses interleave boundary\n"); 2446 return -ENXIO; 2447 } 2448 2449 return 0; 2450 } 2451 2452 static int ars_get_cap(struct acpi_nfit_desc *acpi_desc, 2453 struct nd_cmd_ars_cap *cmd, struct nfit_spa *nfit_spa) 2454 { 2455 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2456 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2457 int cmd_rc, rc; 2458 2459 cmd->address = spa->address; 2460 cmd->length = spa->length; 2461 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, cmd, 2462 sizeof(*cmd), &cmd_rc); 2463 if (rc < 0) 2464 return rc; 2465 return cmd_rc; 2466 } 2467 2468 static int ars_start(struct acpi_nfit_desc *acpi_desc, struct nfit_spa *nfit_spa) 2469 { 2470 int rc; 2471 int cmd_rc; 2472 struct nd_cmd_ars_start ars_start; 2473 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2474 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2475 2476 memset(&ars_start, 0, sizeof(ars_start)); 2477 ars_start.address = spa->address; 2478 ars_start.length = spa->length; 2479 ars_start.flags = acpi_desc->ars_start_flags; 2480 if (nfit_spa_type(spa) == NFIT_SPA_PM) 2481 ars_start.type = ND_ARS_PERSISTENT; 2482 else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE) 2483 ars_start.type = ND_ARS_VOLATILE; 2484 else 2485 return -ENOTTY; 2486 2487 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start, 2488 sizeof(ars_start), &cmd_rc); 2489 2490 if (rc < 0) 2491 return rc; 2492 return cmd_rc; 2493 } 2494 2495 static int ars_continue(struct acpi_nfit_desc *acpi_desc) 2496 { 2497 int rc, cmd_rc; 2498 struct nd_cmd_ars_start ars_start; 2499 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2500 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; 2501 2502 memset(&ars_start, 0, sizeof(ars_start)); 2503 ars_start.address = ars_status->restart_address; 2504 ars_start.length = ars_status->restart_length; 2505 ars_start.type = ars_status->type; 2506 ars_start.flags = acpi_desc->ars_start_flags; 2507 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start, 2508 sizeof(ars_start), &cmd_rc); 2509 if (rc < 0) 2510 return rc; 2511 return cmd_rc; 2512 } 2513 2514 static int ars_get_status(struct acpi_nfit_desc *acpi_desc) 2515 { 2516 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2517 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; 2518 int rc, cmd_rc; 2519 2520 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_STATUS, ars_status, 2521 acpi_desc->ars_status_size, &cmd_rc); 2522 if (rc < 0) 2523 return rc; 2524 return cmd_rc; 2525 } 2526 2527 static int ars_status_process_records(struct acpi_nfit_desc *acpi_desc, 2528 struct nd_cmd_ars_status *ars_status) 2529 { 2530 struct nvdimm_bus *nvdimm_bus = acpi_desc->nvdimm_bus; 2531 int rc; 2532 u32 i; 2533 2534 /* 2535 * First record starts at 44 byte offset from the start of the 2536 * payload. 2537 */ 2538 if (ars_status->out_length < 44) 2539 return 0; 2540 for (i = 0; i < ars_status->num_records; i++) { 2541 /* only process full records */ 2542 if (ars_status->out_length 2543 < 44 + sizeof(struct nd_ars_record) * (i + 1)) 2544 break; 2545 rc = nvdimm_bus_add_badrange(nvdimm_bus, 2546 ars_status->records[i].err_address, 2547 ars_status->records[i].length); 2548 if (rc) 2549 return rc; 2550 } 2551 if (i < ars_status->num_records) 2552 dev_warn(acpi_desc->dev, "detected truncated ars results\n"); 2553 2554 return 0; 2555 } 2556 2557 static void acpi_nfit_remove_resource(void *data) 2558 { 2559 struct resource *res = data; 2560 2561 remove_resource(res); 2562 } 2563 2564 static int acpi_nfit_insert_resource(struct acpi_nfit_desc *acpi_desc, 2565 struct nd_region_desc *ndr_desc) 2566 { 2567 struct resource *res, *nd_res = ndr_desc->res; 2568 int is_pmem, ret; 2569 2570 /* No operation if the region is already registered as PMEM */ 2571 is_pmem = region_intersects(nd_res->start, resource_size(nd_res), 2572 IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY); 2573 if (is_pmem == REGION_INTERSECTS) 2574 return 0; 2575 2576 res = devm_kzalloc(acpi_desc->dev, sizeof(*res), GFP_KERNEL); 2577 if (!res) 2578 return -ENOMEM; 2579 2580 res->name = "Persistent Memory"; 2581 res->start = nd_res->start; 2582 res->end = nd_res->end; 2583 res->flags = IORESOURCE_MEM; 2584 res->desc = IORES_DESC_PERSISTENT_MEMORY; 2585 2586 ret = insert_resource(&iomem_resource, res); 2587 if (ret) 2588 return ret; 2589 2590 ret = devm_add_action_or_reset(acpi_desc->dev, 2591 acpi_nfit_remove_resource, 2592 res); 2593 if (ret) 2594 return ret; 2595 2596 return 0; 2597 } 2598 2599 static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc, 2600 struct nd_mapping_desc *mapping, struct nd_region_desc *ndr_desc, 2601 struct acpi_nfit_memory_map *memdev, 2602 struct nfit_spa *nfit_spa) 2603 { 2604 struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, 2605 memdev->device_handle); 2606 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2607 struct nd_blk_region_desc *ndbr_desc; 2608 struct nfit_mem *nfit_mem; 2609 int blk_valid = 0, rc; 2610 2611 if (!nvdimm) { 2612 dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n", 2613 spa->range_index, memdev->device_handle); 2614 return -ENODEV; 2615 } 2616 2617 mapping->nvdimm = nvdimm; 2618 switch (nfit_spa_type(spa)) { 2619 case NFIT_SPA_PM: 2620 case NFIT_SPA_VOLATILE: 2621 mapping->start = memdev->address; 2622 mapping->size = memdev->region_size; 2623 break; 2624 case NFIT_SPA_DCR: 2625 nfit_mem = nvdimm_provider_data(nvdimm); 2626 if (!nfit_mem || !nfit_mem->bdw) { 2627 dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n", 2628 spa->range_index, nvdimm_name(nvdimm)); 2629 } else { 2630 mapping->size = nfit_mem->bdw->capacity; 2631 mapping->start = nfit_mem->bdw->start_address; 2632 ndr_desc->num_lanes = nfit_mem->bdw->windows; 2633 blk_valid = 1; 2634 } 2635 2636 ndr_desc->mapping = mapping; 2637 ndr_desc->num_mappings = blk_valid; 2638 ndbr_desc = to_blk_region_desc(ndr_desc); 2639 ndbr_desc->enable = acpi_nfit_blk_region_enable; 2640 ndbr_desc->do_io = acpi_desc->blk_do_io; 2641 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa); 2642 if (rc) 2643 return rc; 2644 nfit_spa->nd_region = nvdimm_blk_region_create(acpi_desc->nvdimm_bus, 2645 ndr_desc); 2646 if (!nfit_spa->nd_region) 2647 return -ENOMEM; 2648 break; 2649 } 2650 2651 return 0; 2652 } 2653 2654 static bool nfit_spa_is_virtual(struct acpi_nfit_system_address *spa) 2655 { 2656 return (nfit_spa_type(spa) == NFIT_SPA_VDISK || 2657 nfit_spa_type(spa) == NFIT_SPA_VCD || 2658 nfit_spa_type(spa) == NFIT_SPA_PDISK || 2659 nfit_spa_type(spa) == NFIT_SPA_PCD); 2660 } 2661 2662 static bool nfit_spa_is_volatile(struct acpi_nfit_system_address *spa) 2663 { 2664 return (nfit_spa_type(spa) == NFIT_SPA_VDISK || 2665 nfit_spa_type(spa) == NFIT_SPA_VCD || 2666 nfit_spa_type(spa) == NFIT_SPA_VOLATILE); 2667 } 2668 2669 static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc, 2670 struct nfit_spa *nfit_spa) 2671 { 2672 static struct nd_mapping_desc mappings[ND_MAX_MAPPINGS]; 2673 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2674 struct nd_blk_region_desc ndbr_desc; 2675 struct nd_region_desc *ndr_desc; 2676 struct nfit_memdev *nfit_memdev; 2677 struct nvdimm_bus *nvdimm_bus; 2678 struct resource res; 2679 int count = 0, rc; 2680 2681 if (nfit_spa->nd_region) 2682 return 0; 2683 2684 if (spa->range_index == 0 && !nfit_spa_is_virtual(spa)) { 2685 dev_dbg(acpi_desc->dev, "%s: detected invalid spa index\n", 2686 __func__); 2687 return 0; 2688 } 2689 2690 memset(&res, 0, sizeof(res)); 2691 memset(&mappings, 0, sizeof(mappings)); 2692 memset(&ndbr_desc, 0, sizeof(ndbr_desc)); 2693 res.start = spa->address; 2694 res.end = res.start + spa->length - 1; 2695 ndr_desc = &ndbr_desc.ndr_desc; 2696 ndr_desc->res = &res; 2697 ndr_desc->provider_data = nfit_spa; 2698 ndr_desc->attr_groups = acpi_nfit_region_attribute_groups; 2699 if (spa->flags & ACPI_NFIT_PROXIMITY_VALID) 2700 ndr_desc->numa_node = acpi_map_pxm_to_online_node( 2701 spa->proximity_domain); 2702 else 2703 ndr_desc->numa_node = NUMA_NO_NODE; 2704 2705 /* 2706 * Persistence domain bits are hierarchical, if 2707 * ACPI_NFIT_CAPABILITY_CACHE_FLUSH is set then 2708 * ACPI_NFIT_CAPABILITY_MEM_FLUSH is implied. 2709 */ 2710 if (acpi_desc->platform_cap & ACPI_NFIT_CAPABILITY_CACHE_FLUSH) 2711 set_bit(ND_REGION_PERSIST_CACHE, &ndr_desc->flags); 2712 else if (acpi_desc->platform_cap & ACPI_NFIT_CAPABILITY_MEM_FLUSH) 2713 set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc->flags); 2714 2715 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 2716 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev; 2717 struct nd_mapping_desc *mapping; 2718 2719 if (memdev->range_index != spa->range_index) 2720 continue; 2721 if (count >= ND_MAX_MAPPINGS) { 2722 dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n", 2723 spa->range_index, ND_MAX_MAPPINGS); 2724 return -ENXIO; 2725 } 2726 mapping = &mappings[count++]; 2727 rc = acpi_nfit_init_mapping(acpi_desc, mapping, ndr_desc, 2728 memdev, nfit_spa); 2729 if (rc) 2730 goto out; 2731 } 2732 2733 ndr_desc->mapping = mappings; 2734 ndr_desc->num_mappings = count; 2735 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa); 2736 if (rc) 2737 goto out; 2738 2739 nvdimm_bus = acpi_desc->nvdimm_bus; 2740 if (nfit_spa_type(spa) == NFIT_SPA_PM) { 2741 rc = acpi_nfit_insert_resource(acpi_desc, ndr_desc); 2742 if (rc) { 2743 dev_warn(acpi_desc->dev, 2744 "failed to insert pmem resource to iomem: %d\n", 2745 rc); 2746 goto out; 2747 } 2748 2749 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus, 2750 ndr_desc); 2751 if (!nfit_spa->nd_region) 2752 rc = -ENOMEM; 2753 } else if (nfit_spa_is_volatile(spa)) { 2754 nfit_spa->nd_region = nvdimm_volatile_region_create(nvdimm_bus, 2755 ndr_desc); 2756 if (!nfit_spa->nd_region) 2757 rc = -ENOMEM; 2758 } else if (nfit_spa_is_virtual(spa)) { 2759 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus, 2760 ndr_desc); 2761 if (!nfit_spa->nd_region) 2762 rc = -ENOMEM; 2763 } 2764 2765 out: 2766 if (rc) 2767 dev_err(acpi_desc->dev, "failed to register spa range %d\n", 2768 nfit_spa->spa->range_index); 2769 return rc; 2770 } 2771 2772 static int ars_status_alloc(struct acpi_nfit_desc *acpi_desc, 2773 u32 max_ars) 2774 { 2775 struct device *dev = acpi_desc->dev; 2776 struct nd_cmd_ars_status *ars_status; 2777 2778 if (acpi_desc->ars_status && acpi_desc->ars_status_size >= max_ars) { 2779 memset(acpi_desc->ars_status, 0, acpi_desc->ars_status_size); 2780 return 0; 2781 } 2782 2783 if (acpi_desc->ars_status) 2784 devm_kfree(dev, acpi_desc->ars_status); 2785 acpi_desc->ars_status = NULL; 2786 ars_status = devm_kzalloc(dev, max_ars, GFP_KERNEL); 2787 if (!ars_status) 2788 return -ENOMEM; 2789 acpi_desc->ars_status = ars_status; 2790 acpi_desc->ars_status_size = max_ars; 2791 return 0; 2792 } 2793 2794 static int acpi_nfit_query_poison(struct acpi_nfit_desc *acpi_desc, 2795 struct nfit_spa *nfit_spa) 2796 { 2797 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2798 int rc; 2799 2800 if (!nfit_spa->max_ars) { 2801 struct nd_cmd_ars_cap ars_cap; 2802 2803 memset(&ars_cap, 0, sizeof(ars_cap)); 2804 rc = ars_get_cap(acpi_desc, &ars_cap, nfit_spa); 2805 if (rc < 0) 2806 return rc; 2807 nfit_spa->max_ars = ars_cap.max_ars_out; 2808 nfit_spa->clear_err_unit = ars_cap.clear_err_unit; 2809 /* check that the supported scrub types match the spa type */ 2810 if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE && 2811 ((ars_cap.status >> 16) & ND_ARS_VOLATILE) == 0) 2812 return -ENOTTY; 2813 else if (nfit_spa_type(spa) == NFIT_SPA_PM && 2814 ((ars_cap.status >> 16) & ND_ARS_PERSISTENT) == 0) 2815 return -ENOTTY; 2816 } 2817 2818 if (ars_status_alloc(acpi_desc, nfit_spa->max_ars)) 2819 return -ENOMEM; 2820 2821 rc = ars_get_status(acpi_desc); 2822 if (rc < 0 && rc != -ENOSPC) 2823 return rc; 2824 2825 if (ars_status_process_records(acpi_desc, acpi_desc->ars_status)) 2826 return -ENOMEM; 2827 2828 return 0; 2829 } 2830 2831 static void acpi_nfit_async_scrub(struct acpi_nfit_desc *acpi_desc, 2832 struct nfit_spa *nfit_spa) 2833 { 2834 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2835 unsigned int overflow_retry = scrub_overflow_abort; 2836 u64 init_ars_start = 0, init_ars_len = 0; 2837 struct device *dev = acpi_desc->dev; 2838 unsigned int tmo = scrub_timeout; 2839 int rc; 2840 2841 if (!nfit_spa->ars_required || !nfit_spa->nd_region) 2842 return; 2843 2844 rc = ars_start(acpi_desc, nfit_spa); 2845 /* 2846 * If we timed out the initial scan we'll still be busy here, 2847 * and will wait another timeout before giving up permanently. 2848 */ 2849 if (rc < 0 && rc != -EBUSY) 2850 return; 2851 2852 do { 2853 u64 ars_start, ars_len; 2854 2855 if (acpi_desc->cancel) 2856 break; 2857 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa); 2858 if (rc == -ENOTTY) 2859 break; 2860 if (rc == -EBUSY && !tmo) { 2861 dev_warn(dev, "range %d ars timeout, aborting\n", 2862 spa->range_index); 2863 break; 2864 } 2865 2866 if (rc == -EBUSY) { 2867 /* 2868 * Note, entries may be appended to the list 2869 * while the lock is dropped, but the workqueue 2870 * being active prevents entries being deleted / 2871 * freed. 2872 */ 2873 mutex_unlock(&acpi_desc->init_mutex); 2874 ssleep(1); 2875 tmo--; 2876 mutex_lock(&acpi_desc->init_mutex); 2877 continue; 2878 } 2879 2880 /* we got some results, but there are more pending... */ 2881 if (rc == -ENOSPC && overflow_retry--) { 2882 if (!init_ars_len) { 2883 init_ars_len = acpi_desc->ars_status->length; 2884 init_ars_start = acpi_desc->ars_status->address; 2885 } 2886 rc = ars_continue(acpi_desc); 2887 } 2888 2889 if (rc < 0) { 2890 dev_warn(dev, "range %d ars continuation failed\n", 2891 spa->range_index); 2892 break; 2893 } 2894 2895 if (init_ars_len) { 2896 ars_start = init_ars_start; 2897 ars_len = init_ars_len; 2898 } else { 2899 ars_start = acpi_desc->ars_status->address; 2900 ars_len = acpi_desc->ars_status->length; 2901 } 2902 dev_dbg(dev, "spa range: %d ars from %#llx + %#llx complete\n", 2903 spa->range_index, ars_start, ars_len); 2904 /* notify the region about new poison entries */ 2905 nvdimm_region_notify(nfit_spa->nd_region, 2906 NVDIMM_REVALIDATE_POISON); 2907 break; 2908 } while (1); 2909 } 2910 2911 static void acpi_nfit_scrub(struct work_struct *work) 2912 { 2913 struct device *dev; 2914 u64 init_scrub_length = 0; 2915 struct nfit_spa *nfit_spa; 2916 u64 init_scrub_address = 0; 2917 bool init_ars_done = false; 2918 struct acpi_nfit_desc *acpi_desc; 2919 unsigned int tmo = scrub_timeout; 2920 unsigned int overflow_retry = scrub_overflow_abort; 2921 2922 acpi_desc = container_of(work, typeof(*acpi_desc), work); 2923 dev = acpi_desc->dev; 2924 2925 /* 2926 * We scrub in 2 phases. The first phase waits for any platform 2927 * firmware initiated scrubs to complete and then we go search for the 2928 * affected spa regions to mark them scanned. In the second phase we 2929 * initiate a directed scrub for every range that was not scrubbed in 2930 * phase 1. If we're called for a 'rescan', we harmlessly pass through 2931 * the first phase, but really only care about running phase 2, where 2932 * regions can be notified of new poison. 2933 */ 2934 2935 /* process platform firmware initiated scrubs */ 2936 retry: 2937 mutex_lock(&acpi_desc->init_mutex); 2938 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 2939 struct nd_cmd_ars_status *ars_status; 2940 struct acpi_nfit_system_address *spa; 2941 u64 ars_start, ars_len; 2942 int rc; 2943 2944 if (acpi_desc->cancel) 2945 break; 2946 2947 if (nfit_spa->nd_region) 2948 continue; 2949 2950 if (init_ars_done) { 2951 /* 2952 * No need to re-query, we're now just 2953 * reconciling all the ranges covered by the 2954 * initial scrub 2955 */ 2956 rc = 0; 2957 } else 2958 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa); 2959 2960 if (rc == -ENOTTY) { 2961 /* no ars capability, just register spa and move on */ 2962 acpi_nfit_register_region(acpi_desc, nfit_spa); 2963 continue; 2964 } 2965 2966 if (rc == -EBUSY && !tmo) { 2967 /* fallthrough to directed scrub in phase 2 */ 2968 dev_warn(dev, "timeout awaiting ars results, continuing...\n"); 2969 break; 2970 } else if (rc == -EBUSY) { 2971 mutex_unlock(&acpi_desc->init_mutex); 2972 ssleep(1); 2973 tmo--; 2974 goto retry; 2975 } 2976 2977 /* we got some results, but there are more pending... */ 2978 if (rc == -ENOSPC && overflow_retry--) { 2979 ars_status = acpi_desc->ars_status; 2980 /* 2981 * Record the original scrub range, so that we 2982 * can recall all the ranges impacted by the 2983 * initial scrub. 2984 */ 2985 if (!init_scrub_length) { 2986 init_scrub_length = ars_status->length; 2987 init_scrub_address = ars_status->address; 2988 } 2989 rc = ars_continue(acpi_desc); 2990 if (rc == 0) { 2991 mutex_unlock(&acpi_desc->init_mutex); 2992 goto retry; 2993 } 2994 } 2995 2996 if (rc < 0) { 2997 /* 2998 * Initial scrub failed, we'll give it one more 2999 * try below... 3000 */ 3001 break; 3002 } 3003 3004 /* We got some final results, record completed ranges */ 3005 ars_status = acpi_desc->ars_status; 3006 if (init_scrub_length) { 3007 ars_start = init_scrub_address; 3008 ars_len = ars_start + init_scrub_length; 3009 } else { 3010 ars_start = ars_status->address; 3011 ars_len = ars_status->length; 3012 } 3013 spa = nfit_spa->spa; 3014 3015 if (!init_ars_done) { 3016 init_ars_done = true; 3017 dev_dbg(dev, "init scrub %#llx + %#llx complete\n", 3018 ars_start, ars_len); 3019 } 3020 if (ars_start <= spa->address && ars_start + ars_len 3021 >= spa->address + spa->length) 3022 acpi_nfit_register_region(acpi_desc, nfit_spa); 3023 } 3024 3025 /* 3026 * For all the ranges not covered by an initial scrub we still 3027 * want to see if there are errors, but it's ok to discover them 3028 * asynchronously. 3029 */ 3030 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 3031 /* 3032 * Flag all the ranges that still need scrubbing, but 3033 * register them now to make data available. 3034 */ 3035 if (!nfit_spa->nd_region) { 3036 nfit_spa->ars_required = 1; 3037 acpi_nfit_register_region(acpi_desc, nfit_spa); 3038 } 3039 } 3040 acpi_desc->init_complete = 1; 3041 3042 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) 3043 acpi_nfit_async_scrub(acpi_desc, nfit_spa); 3044 acpi_desc->scrub_count++; 3045 acpi_desc->ars_start_flags = 0; 3046 if (acpi_desc->scrub_count_state) 3047 sysfs_notify_dirent(acpi_desc->scrub_count_state); 3048 mutex_unlock(&acpi_desc->init_mutex); 3049 } 3050 3051 static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc) 3052 { 3053 struct nfit_spa *nfit_spa; 3054 int rc; 3055 3056 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) 3057 if (nfit_spa_type(nfit_spa->spa) == NFIT_SPA_DCR) { 3058 /* BLK regions don't need to wait for ars results */ 3059 rc = acpi_nfit_register_region(acpi_desc, nfit_spa); 3060 if (rc) 3061 return rc; 3062 } 3063 3064 acpi_desc->ars_start_flags = 0; 3065 if (!acpi_desc->cancel) 3066 queue_work(nfit_wq, &acpi_desc->work); 3067 return 0; 3068 } 3069 3070 static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc, 3071 struct nfit_table_prev *prev) 3072 { 3073 struct device *dev = acpi_desc->dev; 3074 3075 if (!list_empty(&prev->spas) || 3076 !list_empty(&prev->memdevs) || 3077 !list_empty(&prev->dcrs) || 3078 !list_empty(&prev->bdws) || 3079 !list_empty(&prev->idts) || 3080 !list_empty(&prev->flushes)) { 3081 dev_err(dev, "new nfit deletes entries (unsupported)\n"); 3082 return -ENXIO; 3083 } 3084 return 0; 3085 } 3086 3087 static int acpi_nfit_desc_init_scrub_attr(struct acpi_nfit_desc *acpi_desc) 3088 { 3089 struct device *dev = acpi_desc->dev; 3090 struct kernfs_node *nfit; 3091 struct device *bus_dev; 3092 3093 if (!ars_supported(acpi_desc->nvdimm_bus)) 3094 return 0; 3095 3096 bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); 3097 nfit = sysfs_get_dirent(bus_dev->kobj.sd, "nfit"); 3098 if (!nfit) { 3099 dev_err(dev, "sysfs_get_dirent 'nfit' failed\n"); 3100 return -ENODEV; 3101 } 3102 acpi_desc->scrub_count_state = sysfs_get_dirent(nfit, "scrub"); 3103 sysfs_put(nfit); 3104 if (!acpi_desc->scrub_count_state) { 3105 dev_err(dev, "sysfs_get_dirent 'scrub' failed\n"); 3106 return -ENODEV; 3107 } 3108 3109 return 0; 3110 } 3111 3112 static void acpi_nfit_unregister(void *data) 3113 { 3114 struct acpi_nfit_desc *acpi_desc = data; 3115 3116 nvdimm_bus_unregister(acpi_desc->nvdimm_bus); 3117 } 3118 3119 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz) 3120 { 3121 struct device *dev = acpi_desc->dev; 3122 struct nfit_table_prev prev; 3123 const void *end; 3124 int rc; 3125 3126 if (!acpi_desc->nvdimm_bus) { 3127 acpi_nfit_init_dsms(acpi_desc); 3128 3129 acpi_desc->nvdimm_bus = nvdimm_bus_register(dev, 3130 &acpi_desc->nd_desc); 3131 if (!acpi_desc->nvdimm_bus) 3132 return -ENOMEM; 3133 3134 rc = devm_add_action_or_reset(dev, acpi_nfit_unregister, 3135 acpi_desc); 3136 if (rc) 3137 return rc; 3138 3139 rc = acpi_nfit_desc_init_scrub_attr(acpi_desc); 3140 if (rc) 3141 return rc; 3142 3143 /* register this acpi_desc for mce notifications */ 3144 mutex_lock(&acpi_desc_lock); 3145 list_add_tail(&acpi_desc->list, &acpi_descs); 3146 mutex_unlock(&acpi_desc_lock); 3147 } 3148 3149 mutex_lock(&acpi_desc->init_mutex); 3150 3151 INIT_LIST_HEAD(&prev.spas); 3152 INIT_LIST_HEAD(&prev.memdevs); 3153 INIT_LIST_HEAD(&prev.dcrs); 3154 INIT_LIST_HEAD(&prev.bdws); 3155 INIT_LIST_HEAD(&prev.idts); 3156 INIT_LIST_HEAD(&prev.flushes); 3157 3158 list_cut_position(&prev.spas, &acpi_desc->spas, 3159 acpi_desc->spas.prev); 3160 list_cut_position(&prev.memdevs, &acpi_desc->memdevs, 3161 acpi_desc->memdevs.prev); 3162 list_cut_position(&prev.dcrs, &acpi_desc->dcrs, 3163 acpi_desc->dcrs.prev); 3164 list_cut_position(&prev.bdws, &acpi_desc->bdws, 3165 acpi_desc->bdws.prev); 3166 list_cut_position(&prev.idts, &acpi_desc->idts, 3167 acpi_desc->idts.prev); 3168 list_cut_position(&prev.flushes, &acpi_desc->flushes, 3169 acpi_desc->flushes.prev); 3170 3171 end = data + sz; 3172 while (!IS_ERR_OR_NULL(data)) 3173 data = add_table(acpi_desc, &prev, data, end); 3174 3175 if (IS_ERR(data)) { 3176 dev_dbg(dev, "%s: nfit table parsing error: %ld\n", __func__, 3177 PTR_ERR(data)); 3178 rc = PTR_ERR(data); 3179 goto out_unlock; 3180 } 3181 3182 rc = acpi_nfit_check_deletions(acpi_desc, &prev); 3183 if (rc) 3184 goto out_unlock; 3185 3186 rc = nfit_mem_init(acpi_desc); 3187 if (rc) 3188 goto out_unlock; 3189 3190 rc = acpi_nfit_register_dimms(acpi_desc); 3191 if (rc) 3192 goto out_unlock; 3193 3194 rc = acpi_nfit_register_regions(acpi_desc); 3195 3196 out_unlock: 3197 mutex_unlock(&acpi_desc->init_mutex); 3198 return rc; 3199 } 3200 EXPORT_SYMBOL_GPL(acpi_nfit_init); 3201 3202 struct acpi_nfit_flush_work { 3203 struct work_struct work; 3204 struct completion cmp; 3205 }; 3206 3207 static void flush_probe(struct work_struct *work) 3208 { 3209 struct acpi_nfit_flush_work *flush; 3210 3211 flush = container_of(work, typeof(*flush), work); 3212 complete(&flush->cmp); 3213 } 3214 3215 static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc) 3216 { 3217 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 3218 struct device *dev = acpi_desc->dev; 3219 struct acpi_nfit_flush_work flush; 3220 int rc; 3221 3222 /* bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */ 3223 device_lock(dev); 3224 device_unlock(dev); 3225 3226 /* bounce the init_mutex to make init_complete valid */ 3227 mutex_lock(&acpi_desc->init_mutex); 3228 if (acpi_desc->cancel || acpi_desc->init_complete) { 3229 mutex_unlock(&acpi_desc->init_mutex); 3230 return 0; 3231 } 3232 3233 /* 3234 * Scrub work could take 10s of seconds, userspace may give up so we 3235 * need to be interruptible while waiting. 3236 */ 3237 INIT_WORK_ONSTACK(&flush.work, flush_probe); 3238 init_completion(&flush.cmp); 3239 queue_work(nfit_wq, &flush.work); 3240 mutex_unlock(&acpi_desc->init_mutex); 3241 3242 rc = wait_for_completion_interruptible(&flush.cmp); 3243 cancel_work_sync(&flush.work); 3244 return rc; 3245 } 3246 3247 static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc, 3248 struct nvdimm *nvdimm, unsigned int cmd) 3249 { 3250 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 3251 3252 if (nvdimm) 3253 return 0; 3254 if (cmd != ND_CMD_ARS_START) 3255 return 0; 3256 3257 /* 3258 * The kernel and userspace may race to initiate a scrub, but 3259 * the scrub thread is prepared to lose that initial race. It 3260 * just needs guarantees that any ars it initiates are not 3261 * interrupted by any intervening start reqeusts from userspace. 3262 */ 3263 if (work_busy(&acpi_desc->work)) 3264 return -EBUSY; 3265 3266 return 0; 3267 } 3268 3269 int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc, u8 flags) 3270 { 3271 struct device *dev = acpi_desc->dev; 3272 struct nfit_spa *nfit_spa; 3273 3274 if (work_busy(&acpi_desc->work)) 3275 return -EBUSY; 3276 3277 mutex_lock(&acpi_desc->init_mutex); 3278 if (acpi_desc->cancel) { 3279 mutex_unlock(&acpi_desc->init_mutex); 3280 return 0; 3281 } 3282 3283 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 3284 struct acpi_nfit_system_address *spa = nfit_spa->spa; 3285 3286 if (nfit_spa_type(spa) != NFIT_SPA_PM) 3287 continue; 3288 3289 nfit_spa->ars_required = 1; 3290 } 3291 acpi_desc->ars_start_flags = flags; 3292 queue_work(nfit_wq, &acpi_desc->work); 3293 dev_dbg(dev, "%s: ars_scan triggered\n", __func__); 3294 mutex_unlock(&acpi_desc->init_mutex); 3295 3296 return 0; 3297 } 3298 3299 void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev) 3300 { 3301 struct nvdimm_bus_descriptor *nd_desc; 3302 3303 dev_set_drvdata(dev, acpi_desc); 3304 acpi_desc->dev = dev; 3305 acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io; 3306 nd_desc = &acpi_desc->nd_desc; 3307 nd_desc->provider_name = "ACPI.NFIT"; 3308 nd_desc->module = THIS_MODULE; 3309 nd_desc->ndctl = acpi_nfit_ctl; 3310 nd_desc->flush_probe = acpi_nfit_flush_probe; 3311 nd_desc->clear_to_send = acpi_nfit_clear_to_send; 3312 nd_desc->attr_groups = acpi_nfit_attribute_groups; 3313 3314 INIT_LIST_HEAD(&acpi_desc->spas); 3315 INIT_LIST_HEAD(&acpi_desc->dcrs); 3316 INIT_LIST_HEAD(&acpi_desc->bdws); 3317 INIT_LIST_HEAD(&acpi_desc->idts); 3318 INIT_LIST_HEAD(&acpi_desc->flushes); 3319 INIT_LIST_HEAD(&acpi_desc->memdevs); 3320 INIT_LIST_HEAD(&acpi_desc->dimms); 3321 INIT_LIST_HEAD(&acpi_desc->list); 3322 mutex_init(&acpi_desc->init_mutex); 3323 INIT_WORK(&acpi_desc->work, acpi_nfit_scrub); 3324 } 3325 EXPORT_SYMBOL_GPL(acpi_nfit_desc_init); 3326 3327 static void acpi_nfit_put_table(void *table) 3328 { 3329 acpi_put_table(table); 3330 } 3331 3332 void acpi_nfit_shutdown(void *data) 3333 { 3334 struct acpi_nfit_desc *acpi_desc = data; 3335 struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); 3336 3337 /* 3338 * Destruct under acpi_desc_lock so that nfit_handle_mce does not 3339 * race teardown 3340 */ 3341 mutex_lock(&acpi_desc_lock); 3342 list_del(&acpi_desc->list); 3343 mutex_unlock(&acpi_desc_lock); 3344 3345 mutex_lock(&acpi_desc->init_mutex); 3346 acpi_desc->cancel = 1; 3347 mutex_unlock(&acpi_desc->init_mutex); 3348 3349 /* 3350 * Bounce the nvdimm bus lock to make sure any in-flight 3351 * acpi_nfit_ars_rescan() submissions have had a chance to 3352 * either submit or see ->cancel set. 3353 */ 3354 device_lock(bus_dev); 3355 device_unlock(bus_dev); 3356 3357 flush_workqueue(nfit_wq); 3358 } 3359 EXPORT_SYMBOL_GPL(acpi_nfit_shutdown); 3360 3361 static int acpi_nfit_add(struct acpi_device *adev) 3362 { 3363 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 3364 struct acpi_nfit_desc *acpi_desc; 3365 struct device *dev = &adev->dev; 3366 struct acpi_table_header *tbl; 3367 acpi_status status = AE_OK; 3368 acpi_size sz; 3369 int rc = 0; 3370 3371 status = acpi_get_table(ACPI_SIG_NFIT, 0, &tbl); 3372 if (ACPI_FAILURE(status)) { 3373 /* This is ok, we could have an nvdimm hotplugged later */ 3374 dev_dbg(dev, "failed to find NFIT at startup\n"); 3375 return 0; 3376 } 3377 3378 rc = devm_add_action_or_reset(dev, acpi_nfit_put_table, tbl); 3379 if (rc) 3380 return rc; 3381 sz = tbl->length; 3382 3383 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL); 3384 if (!acpi_desc) 3385 return -ENOMEM; 3386 acpi_nfit_desc_init(acpi_desc, &adev->dev); 3387 3388 /* Save the acpi header for exporting the revision via sysfs */ 3389 acpi_desc->acpi_header = *tbl; 3390 3391 /* Evaluate _FIT and override with that if present */ 3392 status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf); 3393 if (ACPI_SUCCESS(status) && buf.length > 0) { 3394 union acpi_object *obj = buf.pointer; 3395 3396 if (obj->type == ACPI_TYPE_BUFFER) 3397 rc = acpi_nfit_init(acpi_desc, obj->buffer.pointer, 3398 obj->buffer.length); 3399 else 3400 dev_dbg(dev, "%s invalid type %d, ignoring _FIT\n", 3401 __func__, (int) obj->type); 3402 kfree(buf.pointer); 3403 } else 3404 /* skip over the lead-in header table */ 3405 rc = acpi_nfit_init(acpi_desc, (void *) tbl 3406 + sizeof(struct acpi_table_nfit), 3407 sz - sizeof(struct acpi_table_nfit)); 3408 3409 if (rc) 3410 return rc; 3411 return devm_add_action_or_reset(dev, acpi_nfit_shutdown, acpi_desc); 3412 } 3413 3414 static int acpi_nfit_remove(struct acpi_device *adev) 3415 { 3416 /* see acpi_nfit_unregister */ 3417 return 0; 3418 } 3419 3420 static void acpi_nfit_update_notify(struct device *dev, acpi_handle handle) 3421 { 3422 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev); 3423 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 3424 union acpi_object *obj; 3425 acpi_status status; 3426 int ret; 3427 3428 if (!dev->driver) { 3429 /* dev->driver may be null if we're being removed */ 3430 dev_dbg(dev, "%s: no driver found for dev\n", __func__); 3431 return; 3432 } 3433 3434 if (!acpi_desc) { 3435 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL); 3436 if (!acpi_desc) 3437 return; 3438 acpi_nfit_desc_init(acpi_desc, dev); 3439 } else { 3440 /* 3441 * Finish previous registration before considering new 3442 * regions. 3443 */ 3444 flush_workqueue(nfit_wq); 3445 } 3446 3447 /* Evaluate _FIT */ 3448 status = acpi_evaluate_object(handle, "_FIT", NULL, &buf); 3449 if (ACPI_FAILURE(status)) { 3450 dev_err(dev, "failed to evaluate _FIT\n"); 3451 return; 3452 } 3453 3454 obj = buf.pointer; 3455 if (obj->type == ACPI_TYPE_BUFFER) { 3456 ret = acpi_nfit_init(acpi_desc, obj->buffer.pointer, 3457 obj->buffer.length); 3458 if (ret) 3459 dev_err(dev, "failed to merge updated NFIT\n"); 3460 } else 3461 dev_err(dev, "Invalid _FIT\n"); 3462 kfree(buf.pointer); 3463 } 3464 3465 static void acpi_nfit_uc_error_notify(struct device *dev, acpi_handle handle) 3466 { 3467 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev); 3468 u8 flags = (acpi_desc->scrub_mode == HW_ERROR_SCRUB_ON) ? 3469 0 : ND_ARS_RETURN_PREV_DATA; 3470 3471 acpi_nfit_ars_rescan(acpi_desc, flags); 3472 } 3473 3474 void __acpi_nfit_notify(struct device *dev, acpi_handle handle, u32 event) 3475 { 3476 dev_dbg(dev, "%s: event: 0x%x\n", __func__, event); 3477 3478 switch (event) { 3479 case NFIT_NOTIFY_UPDATE: 3480 return acpi_nfit_update_notify(dev, handle); 3481 case NFIT_NOTIFY_UC_MEMORY_ERROR: 3482 return acpi_nfit_uc_error_notify(dev, handle); 3483 default: 3484 return; 3485 } 3486 } 3487 EXPORT_SYMBOL_GPL(__acpi_nfit_notify); 3488 3489 static void acpi_nfit_notify(struct acpi_device *adev, u32 event) 3490 { 3491 device_lock(&adev->dev); 3492 __acpi_nfit_notify(&adev->dev, adev->handle, event); 3493 device_unlock(&adev->dev); 3494 } 3495 3496 static const struct acpi_device_id acpi_nfit_ids[] = { 3497 { "ACPI0012", 0 }, 3498 { "", 0 }, 3499 }; 3500 MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids); 3501 3502 static struct acpi_driver acpi_nfit_driver = { 3503 .name = KBUILD_MODNAME, 3504 .ids = acpi_nfit_ids, 3505 .ops = { 3506 .add = acpi_nfit_add, 3507 .remove = acpi_nfit_remove, 3508 .notify = acpi_nfit_notify, 3509 }, 3510 }; 3511 3512 static __init int nfit_init(void) 3513 { 3514 int ret; 3515 3516 BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40); 3517 BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56); 3518 BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48); 3519 BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20); 3520 BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9); 3521 BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80); 3522 BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40); 3523 BUILD_BUG_ON(sizeof(struct acpi_nfit_capabilities) != 16); 3524 3525 guid_parse(UUID_VOLATILE_MEMORY, &nfit_uuid[NFIT_SPA_VOLATILE]); 3526 guid_parse(UUID_PERSISTENT_MEMORY, &nfit_uuid[NFIT_SPA_PM]); 3527 guid_parse(UUID_CONTROL_REGION, &nfit_uuid[NFIT_SPA_DCR]); 3528 guid_parse(UUID_DATA_REGION, &nfit_uuid[NFIT_SPA_BDW]); 3529 guid_parse(UUID_VOLATILE_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_VDISK]); 3530 guid_parse(UUID_VOLATILE_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_VCD]); 3531 guid_parse(UUID_PERSISTENT_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_PDISK]); 3532 guid_parse(UUID_PERSISTENT_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_PCD]); 3533 guid_parse(UUID_NFIT_BUS, &nfit_uuid[NFIT_DEV_BUS]); 3534 guid_parse(UUID_NFIT_DIMM, &nfit_uuid[NFIT_DEV_DIMM]); 3535 guid_parse(UUID_NFIT_DIMM_N_HPE1, &nfit_uuid[NFIT_DEV_DIMM_N_HPE1]); 3536 guid_parse(UUID_NFIT_DIMM_N_HPE2, &nfit_uuid[NFIT_DEV_DIMM_N_HPE2]); 3537 guid_parse(UUID_NFIT_DIMM_N_MSFT, &nfit_uuid[NFIT_DEV_DIMM_N_MSFT]); 3538 3539 nfit_wq = create_singlethread_workqueue("nfit"); 3540 if (!nfit_wq) 3541 return -ENOMEM; 3542 3543 nfit_mce_register(); 3544 ret = acpi_bus_register_driver(&acpi_nfit_driver); 3545 if (ret) { 3546 nfit_mce_unregister(); 3547 destroy_workqueue(nfit_wq); 3548 } 3549 3550 return ret; 3551 3552 } 3553 3554 static __exit void nfit_exit(void) 3555 { 3556 nfit_mce_unregister(); 3557 acpi_bus_unregister_driver(&acpi_nfit_driver); 3558 destroy_workqueue(nfit_wq); 3559 WARN_ON(!list_empty(&acpi_descs)); 3560 } 3561 3562 module_init(nfit_init); 3563 module_exit(nfit_exit); 3564 MODULE_LICENSE("GPL v2"); 3565 MODULE_AUTHOR("Intel Corporation"); 3566