1 /* 2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of version 2 of the GNU General Public License as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 */ 13 #include <linux/list_sort.h> 14 #include <linux/libnvdimm.h> 15 #include <linux/module.h> 16 #include <linux/mutex.h> 17 #include <linux/ndctl.h> 18 #include <linux/sysfs.h> 19 #include <linux/delay.h> 20 #include <linux/list.h> 21 #include <linux/acpi.h> 22 #include <linux/sort.h> 23 #include <linux/pmem.h> 24 #include <linux/io.h> 25 #include <linux/nd.h> 26 #include <asm/cacheflush.h> 27 #include "nfit.h" 28 29 /* 30 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is 31 * irrelevant. 32 */ 33 #include <linux/io-64-nonatomic-hi-lo.h> 34 35 static bool force_enable_dimms; 36 module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR); 37 MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status"); 38 39 static unsigned int scrub_timeout = NFIT_ARS_TIMEOUT; 40 module_param(scrub_timeout, uint, S_IRUGO|S_IWUSR); 41 MODULE_PARM_DESC(scrub_timeout, "Initial scrub timeout in seconds"); 42 43 /* after three payloads of overflow, it's dead jim */ 44 static unsigned int scrub_overflow_abort = 3; 45 module_param(scrub_overflow_abort, uint, S_IRUGO|S_IWUSR); 46 MODULE_PARM_DESC(scrub_overflow_abort, 47 "Number of times we overflow ARS results before abort"); 48 49 static bool disable_vendor_specific; 50 module_param(disable_vendor_specific, bool, S_IRUGO); 51 MODULE_PARM_DESC(disable_vendor_specific, 52 "Limit commands to the publicly specified set\n"); 53 54 LIST_HEAD(acpi_descs); 55 DEFINE_MUTEX(acpi_desc_lock); 56 57 static struct workqueue_struct *nfit_wq; 58 59 struct nfit_table_prev { 60 struct list_head spas; 61 struct list_head memdevs; 62 struct list_head dcrs; 63 struct list_head bdws; 64 struct list_head idts; 65 struct list_head flushes; 66 }; 67 68 static u8 nfit_uuid[NFIT_UUID_MAX][16]; 69 70 const u8 *to_nfit_uuid(enum nfit_uuids id) 71 { 72 return nfit_uuid[id]; 73 } 74 EXPORT_SYMBOL(to_nfit_uuid); 75 76 static struct acpi_nfit_desc *to_acpi_nfit_desc( 77 struct nvdimm_bus_descriptor *nd_desc) 78 { 79 return container_of(nd_desc, struct acpi_nfit_desc, nd_desc); 80 } 81 82 static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc) 83 { 84 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 85 86 /* 87 * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct 88 * acpi_device. 89 */ 90 if (!nd_desc->provider_name 91 || strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0) 92 return NULL; 93 94 return to_acpi_device(acpi_desc->dev); 95 } 96 97 static int xlat_status(void *buf, unsigned int cmd, u32 status) 98 { 99 struct nd_cmd_clear_error *clear_err; 100 struct nd_cmd_ars_status *ars_status; 101 u16 flags; 102 103 switch (cmd) { 104 case ND_CMD_ARS_CAP: 105 if ((status & 0xffff) == NFIT_ARS_CAP_NONE) 106 return -ENOTTY; 107 108 /* Command failed */ 109 if (status & 0xffff) 110 return -EIO; 111 112 /* No supported scan types for this range */ 113 flags = ND_ARS_PERSISTENT | ND_ARS_VOLATILE; 114 if ((status >> 16 & flags) == 0) 115 return -ENOTTY; 116 break; 117 case ND_CMD_ARS_START: 118 /* ARS is in progress */ 119 if ((status & 0xffff) == NFIT_ARS_START_BUSY) 120 return -EBUSY; 121 122 /* Command failed */ 123 if (status & 0xffff) 124 return -EIO; 125 break; 126 case ND_CMD_ARS_STATUS: 127 ars_status = buf; 128 /* Command failed */ 129 if (status & 0xffff) 130 return -EIO; 131 /* Check extended status (Upper two bytes) */ 132 if (status == NFIT_ARS_STATUS_DONE) 133 return 0; 134 135 /* ARS is in progress */ 136 if (status == NFIT_ARS_STATUS_BUSY) 137 return -EBUSY; 138 139 /* No ARS performed for the current boot */ 140 if (status == NFIT_ARS_STATUS_NONE) 141 return -EAGAIN; 142 143 /* 144 * ARS interrupted, either we overflowed or some other 145 * agent wants the scan to stop. If we didn't overflow 146 * then just continue with the returned results. 147 */ 148 if (status == NFIT_ARS_STATUS_INTR) { 149 if (ars_status->flags & NFIT_ARS_F_OVERFLOW) 150 return -ENOSPC; 151 return 0; 152 } 153 154 /* Unknown status */ 155 if (status >> 16) 156 return -EIO; 157 break; 158 case ND_CMD_CLEAR_ERROR: 159 clear_err = buf; 160 if (status & 0xffff) 161 return -EIO; 162 if (!clear_err->cleared) 163 return -EIO; 164 if (clear_err->length > clear_err->cleared) 165 return clear_err->cleared; 166 break; 167 default: 168 break; 169 } 170 171 /* all other non-zero status results in an error */ 172 if (status) 173 return -EIO; 174 return 0; 175 } 176 177 static int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, 178 struct nvdimm *nvdimm, unsigned int cmd, void *buf, 179 unsigned int buf_len, int *cmd_rc) 180 { 181 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 182 union acpi_object in_obj, in_buf, *out_obj; 183 const struct nd_cmd_desc *desc = NULL; 184 struct device *dev = acpi_desc->dev; 185 struct nd_cmd_pkg *call_pkg = NULL; 186 const char *cmd_name, *dimm_name; 187 unsigned long cmd_mask, dsm_mask; 188 u32 offset, fw_status = 0; 189 acpi_handle handle; 190 unsigned int func; 191 const u8 *uuid; 192 int rc, i; 193 194 func = cmd; 195 if (cmd == ND_CMD_CALL) { 196 call_pkg = buf; 197 func = call_pkg->nd_command; 198 } 199 200 if (nvdimm) { 201 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 202 struct acpi_device *adev = nfit_mem->adev; 203 204 if (!adev) 205 return -ENOTTY; 206 if (call_pkg && nfit_mem->family != call_pkg->nd_family) 207 return -ENOTTY; 208 209 dimm_name = nvdimm_name(nvdimm); 210 cmd_name = nvdimm_cmd_name(cmd); 211 cmd_mask = nvdimm_cmd_mask(nvdimm); 212 dsm_mask = nfit_mem->dsm_mask; 213 desc = nd_cmd_dimm_desc(cmd); 214 uuid = to_nfit_uuid(nfit_mem->family); 215 handle = adev->handle; 216 } else { 217 struct acpi_device *adev = to_acpi_dev(acpi_desc); 218 219 cmd_name = nvdimm_bus_cmd_name(cmd); 220 cmd_mask = nd_desc->cmd_mask; 221 dsm_mask = cmd_mask; 222 desc = nd_cmd_bus_desc(cmd); 223 uuid = to_nfit_uuid(NFIT_DEV_BUS); 224 handle = adev->handle; 225 dimm_name = "bus"; 226 } 227 228 if (!desc || (cmd && (desc->out_num + desc->in_num == 0))) 229 return -ENOTTY; 230 231 if (!test_bit(cmd, &cmd_mask) || !test_bit(func, &dsm_mask)) 232 return -ENOTTY; 233 234 in_obj.type = ACPI_TYPE_PACKAGE; 235 in_obj.package.count = 1; 236 in_obj.package.elements = &in_buf; 237 in_buf.type = ACPI_TYPE_BUFFER; 238 in_buf.buffer.pointer = buf; 239 in_buf.buffer.length = 0; 240 241 /* libnvdimm has already validated the input envelope */ 242 for (i = 0; i < desc->in_num; i++) 243 in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc, 244 i, buf); 245 246 if (call_pkg) { 247 /* skip over package wrapper */ 248 in_buf.buffer.pointer = (void *) &call_pkg->nd_payload; 249 in_buf.buffer.length = call_pkg->nd_size_in; 250 } 251 252 if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) { 253 dev_dbg(dev, "%s:%s cmd: %d: func: %d input length: %d\n", 254 __func__, dimm_name, cmd, func, 255 in_buf.buffer.length); 256 print_hex_dump_debug("nvdimm in ", DUMP_PREFIX_OFFSET, 4, 4, 257 in_buf.buffer.pointer, 258 min_t(u32, 256, in_buf.buffer.length), true); 259 } 260 261 out_obj = acpi_evaluate_dsm(handle, uuid, 1, func, &in_obj); 262 if (!out_obj) { 263 dev_dbg(dev, "%s:%s _DSM failed cmd: %s\n", __func__, dimm_name, 264 cmd_name); 265 return -EINVAL; 266 } 267 268 if (call_pkg) { 269 call_pkg->nd_fw_size = out_obj->buffer.length; 270 memcpy(call_pkg->nd_payload + call_pkg->nd_size_in, 271 out_obj->buffer.pointer, 272 min(call_pkg->nd_fw_size, call_pkg->nd_size_out)); 273 274 ACPI_FREE(out_obj); 275 /* 276 * Need to support FW function w/o known size in advance. 277 * Caller can determine required size based upon nd_fw_size. 278 * If we return an error (like elsewhere) then caller wouldn't 279 * be able to rely upon data returned to make calculation. 280 */ 281 return 0; 282 } 283 284 if (out_obj->package.type != ACPI_TYPE_BUFFER) { 285 dev_dbg(dev, "%s:%s unexpected output object type cmd: %s type: %d\n", 286 __func__, dimm_name, cmd_name, out_obj->type); 287 rc = -EINVAL; 288 goto out; 289 } 290 291 if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) { 292 dev_dbg(dev, "%s:%s cmd: %s output length: %d\n", __func__, 293 dimm_name, cmd_name, out_obj->buffer.length); 294 print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4, 295 4, out_obj->buffer.pointer, min_t(u32, 128, 296 out_obj->buffer.length), true); 297 } 298 299 for (i = 0, offset = 0; i < desc->out_num; i++) { 300 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf, 301 (u32 *) out_obj->buffer.pointer); 302 303 if (offset + out_size > out_obj->buffer.length) { 304 dev_dbg(dev, "%s:%s output object underflow cmd: %s field: %d\n", 305 __func__, dimm_name, cmd_name, i); 306 break; 307 } 308 309 if (in_buf.buffer.length + offset + out_size > buf_len) { 310 dev_dbg(dev, "%s:%s output overrun cmd: %s field: %d\n", 311 __func__, dimm_name, cmd_name, i); 312 rc = -ENXIO; 313 goto out; 314 } 315 memcpy(buf + in_buf.buffer.length + offset, 316 out_obj->buffer.pointer + offset, out_size); 317 offset += out_size; 318 } 319 320 /* 321 * Set fw_status for all the commands with a known format to be 322 * later interpreted by xlat_status(). 323 */ 324 if (i >= 1 && ((cmd >= ND_CMD_ARS_CAP && cmd <= ND_CMD_CLEAR_ERROR) 325 || (cmd >= ND_CMD_SMART && cmd <= ND_CMD_VENDOR))) 326 fw_status = *(u32 *) out_obj->buffer.pointer; 327 328 if (offset + in_buf.buffer.length < buf_len) { 329 if (i >= 1) { 330 /* 331 * status valid, return the number of bytes left 332 * unfilled in the output buffer 333 */ 334 rc = buf_len - offset - in_buf.buffer.length; 335 if (cmd_rc) 336 *cmd_rc = xlat_status(buf, cmd, fw_status); 337 } else { 338 dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n", 339 __func__, dimm_name, cmd_name, buf_len, 340 offset); 341 rc = -ENXIO; 342 } 343 } else { 344 rc = 0; 345 if (cmd_rc) 346 *cmd_rc = xlat_status(buf, cmd, fw_status); 347 } 348 349 out: 350 ACPI_FREE(out_obj); 351 352 return rc; 353 } 354 355 static const char *spa_type_name(u16 type) 356 { 357 static const char *to_name[] = { 358 [NFIT_SPA_VOLATILE] = "volatile", 359 [NFIT_SPA_PM] = "pmem", 360 [NFIT_SPA_DCR] = "dimm-control-region", 361 [NFIT_SPA_BDW] = "block-data-window", 362 [NFIT_SPA_VDISK] = "volatile-disk", 363 [NFIT_SPA_VCD] = "volatile-cd", 364 [NFIT_SPA_PDISK] = "persistent-disk", 365 [NFIT_SPA_PCD] = "persistent-cd", 366 367 }; 368 369 if (type > NFIT_SPA_PCD) 370 return "unknown"; 371 372 return to_name[type]; 373 } 374 375 int nfit_spa_type(struct acpi_nfit_system_address *spa) 376 { 377 int i; 378 379 for (i = 0; i < NFIT_UUID_MAX; i++) 380 if (memcmp(to_nfit_uuid(i), spa->range_guid, 16) == 0) 381 return i; 382 return -1; 383 } 384 385 static bool add_spa(struct acpi_nfit_desc *acpi_desc, 386 struct nfit_table_prev *prev, 387 struct acpi_nfit_system_address *spa) 388 { 389 struct device *dev = acpi_desc->dev; 390 struct nfit_spa *nfit_spa; 391 392 if (spa->header.length != sizeof(*spa)) 393 return false; 394 395 list_for_each_entry(nfit_spa, &prev->spas, list) { 396 if (memcmp(nfit_spa->spa, spa, sizeof(*spa)) == 0) { 397 list_move_tail(&nfit_spa->list, &acpi_desc->spas); 398 return true; 399 } 400 } 401 402 nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa) + sizeof(*spa), 403 GFP_KERNEL); 404 if (!nfit_spa) 405 return false; 406 INIT_LIST_HEAD(&nfit_spa->list); 407 memcpy(nfit_spa->spa, spa, sizeof(*spa)); 408 list_add_tail(&nfit_spa->list, &acpi_desc->spas); 409 dev_dbg(dev, "%s: spa index: %d type: %s\n", __func__, 410 spa->range_index, 411 spa_type_name(nfit_spa_type(spa))); 412 return true; 413 } 414 415 static bool add_memdev(struct acpi_nfit_desc *acpi_desc, 416 struct nfit_table_prev *prev, 417 struct acpi_nfit_memory_map *memdev) 418 { 419 struct device *dev = acpi_desc->dev; 420 struct nfit_memdev *nfit_memdev; 421 422 if (memdev->header.length != sizeof(*memdev)) 423 return false; 424 425 list_for_each_entry(nfit_memdev, &prev->memdevs, list) 426 if (memcmp(nfit_memdev->memdev, memdev, sizeof(*memdev)) == 0) { 427 list_move_tail(&nfit_memdev->list, &acpi_desc->memdevs); 428 return true; 429 } 430 431 nfit_memdev = devm_kzalloc(dev, sizeof(*nfit_memdev) + sizeof(*memdev), 432 GFP_KERNEL); 433 if (!nfit_memdev) 434 return false; 435 INIT_LIST_HEAD(&nfit_memdev->list); 436 memcpy(nfit_memdev->memdev, memdev, sizeof(*memdev)); 437 list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs); 438 dev_dbg(dev, "%s: memdev handle: %#x spa: %d dcr: %d\n", 439 __func__, memdev->device_handle, memdev->range_index, 440 memdev->region_index); 441 return true; 442 } 443 444 /* 445 * An implementation may provide a truncated control region if no block windows 446 * are defined. 447 */ 448 static size_t sizeof_dcr(struct acpi_nfit_control_region *dcr) 449 { 450 if (dcr->header.length < offsetof(struct acpi_nfit_control_region, 451 window_size)) 452 return 0; 453 if (dcr->windows) 454 return sizeof(*dcr); 455 return offsetof(struct acpi_nfit_control_region, window_size); 456 } 457 458 static bool add_dcr(struct acpi_nfit_desc *acpi_desc, 459 struct nfit_table_prev *prev, 460 struct acpi_nfit_control_region *dcr) 461 { 462 struct device *dev = acpi_desc->dev; 463 struct nfit_dcr *nfit_dcr; 464 465 if (!sizeof_dcr(dcr)) 466 return false; 467 468 list_for_each_entry(nfit_dcr, &prev->dcrs, list) 469 if (memcmp(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)) == 0) { 470 list_move_tail(&nfit_dcr->list, &acpi_desc->dcrs); 471 return true; 472 } 473 474 nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr) + sizeof(*dcr), 475 GFP_KERNEL); 476 if (!nfit_dcr) 477 return false; 478 INIT_LIST_HEAD(&nfit_dcr->list); 479 memcpy(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)); 480 list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs); 481 dev_dbg(dev, "%s: dcr index: %d windows: %d\n", __func__, 482 dcr->region_index, dcr->windows); 483 return true; 484 } 485 486 static bool add_bdw(struct acpi_nfit_desc *acpi_desc, 487 struct nfit_table_prev *prev, 488 struct acpi_nfit_data_region *bdw) 489 { 490 struct device *dev = acpi_desc->dev; 491 struct nfit_bdw *nfit_bdw; 492 493 if (bdw->header.length != sizeof(*bdw)) 494 return false; 495 list_for_each_entry(nfit_bdw, &prev->bdws, list) 496 if (memcmp(nfit_bdw->bdw, bdw, sizeof(*bdw)) == 0) { 497 list_move_tail(&nfit_bdw->list, &acpi_desc->bdws); 498 return true; 499 } 500 501 nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw) + sizeof(*bdw), 502 GFP_KERNEL); 503 if (!nfit_bdw) 504 return false; 505 INIT_LIST_HEAD(&nfit_bdw->list); 506 memcpy(nfit_bdw->bdw, bdw, sizeof(*bdw)); 507 list_add_tail(&nfit_bdw->list, &acpi_desc->bdws); 508 dev_dbg(dev, "%s: bdw dcr: %d windows: %d\n", __func__, 509 bdw->region_index, bdw->windows); 510 return true; 511 } 512 513 static size_t sizeof_idt(struct acpi_nfit_interleave *idt) 514 { 515 if (idt->header.length < sizeof(*idt)) 516 return 0; 517 return sizeof(*idt) + sizeof(u32) * (idt->line_count - 1); 518 } 519 520 static bool add_idt(struct acpi_nfit_desc *acpi_desc, 521 struct nfit_table_prev *prev, 522 struct acpi_nfit_interleave *idt) 523 { 524 struct device *dev = acpi_desc->dev; 525 struct nfit_idt *nfit_idt; 526 527 if (!sizeof_idt(idt)) 528 return false; 529 530 list_for_each_entry(nfit_idt, &prev->idts, list) { 531 if (sizeof_idt(nfit_idt->idt) != sizeof_idt(idt)) 532 continue; 533 534 if (memcmp(nfit_idt->idt, idt, sizeof_idt(idt)) == 0) { 535 list_move_tail(&nfit_idt->list, &acpi_desc->idts); 536 return true; 537 } 538 } 539 540 nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt) + sizeof_idt(idt), 541 GFP_KERNEL); 542 if (!nfit_idt) 543 return false; 544 INIT_LIST_HEAD(&nfit_idt->list); 545 memcpy(nfit_idt->idt, idt, sizeof_idt(idt)); 546 list_add_tail(&nfit_idt->list, &acpi_desc->idts); 547 dev_dbg(dev, "%s: idt index: %d num_lines: %d\n", __func__, 548 idt->interleave_index, idt->line_count); 549 return true; 550 } 551 552 static size_t sizeof_flush(struct acpi_nfit_flush_address *flush) 553 { 554 if (flush->header.length < sizeof(*flush)) 555 return 0; 556 return sizeof(*flush) + sizeof(u64) * (flush->hint_count - 1); 557 } 558 559 static bool add_flush(struct acpi_nfit_desc *acpi_desc, 560 struct nfit_table_prev *prev, 561 struct acpi_nfit_flush_address *flush) 562 { 563 struct device *dev = acpi_desc->dev; 564 struct nfit_flush *nfit_flush; 565 566 if (!sizeof_flush(flush)) 567 return false; 568 569 list_for_each_entry(nfit_flush, &prev->flushes, list) { 570 if (sizeof_flush(nfit_flush->flush) != sizeof_flush(flush)) 571 continue; 572 573 if (memcmp(nfit_flush->flush, flush, 574 sizeof_flush(flush)) == 0) { 575 list_move_tail(&nfit_flush->list, &acpi_desc->flushes); 576 return true; 577 } 578 } 579 580 nfit_flush = devm_kzalloc(dev, sizeof(*nfit_flush) 581 + sizeof_flush(flush), GFP_KERNEL); 582 if (!nfit_flush) 583 return false; 584 INIT_LIST_HEAD(&nfit_flush->list); 585 memcpy(nfit_flush->flush, flush, sizeof_flush(flush)); 586 list_add_tail(&nfit_flush->list, &acpi_desc->flushes); 587 dev_dbg(dev, "%s: nfit_flush handle: %d hint_count: %d\n", __func__, 588 flush->device_handle, flush->hint_count); 589 return true; 590 } 591 592 static void *add_table(struct acpi_nfit_desc *acpi_desc, 593 struct nfit_table_prev *prev, void *table, const void *end) 594 { 595 struct device *dev = acpi_desc->dev; 596 struct acpi_nfit_header *hdr; 597 void *err = ERR_PTR(-ENOMEM); 598 599 if (table >= end) 600 return NULL; 601 602 hdr = table; 603 if (!hdr->length) { 604 dev_warn(dev, "found a zero length table '%d' parsing nfit\n", 605 hdr->type); 606 return NULL; 607 } 608 609 switch (hdr->type) { 610 case ACPI_NFIT_TYPE_SYSTEM_ADDRESS: 611 if (!add_spa(acpi_desc, prev, table)) 612 return err; 613 break; 614 case ACPI_NFIT_TYPE_MEMORY_MAP: 615 if (!add_memdev(acpi_desc, prev, table)) 616 return err; 617 break; 618 case ACPI_NFIT_TYPE_CONTROL_REGION: 619 if (!add_dcr(acpi_desc, prev, table)) 620 return err; 621 break; 622 case ACPI_NFIT_TYPE_DATA_REGION: 623 if (!add_bdw(acpi_desc, prev, table)) 624 return err; 625 break; 626 case ACPI_NFIT_TYPE_INTERLEAVE: 627 if (!add_idt(acpi_desc, prev, table)) 628 return err; 629 break; 630 case ACPI_NFIT_TYPE_FLUSH_ADDRESS: 631 if (!add_flush(acpi_desc, prev, table)) 632 return err; 633 break; 634 case ACPI_NFIT_TYPE_SMBIOS: 635 dev_dbg(dev, "%s: smbios\n", __func__); 636 break; 637 default: 638 dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type); 639 break; 640 } 641 642 return table + hdr->length; 643 } 644 645 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc, 646 struct nfit_mem *nfit_mem) 647 { 648 u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle; 649 u16 dcr = nfit_mem->dcr->region_index; 650 struct nfit_spa *nfit_spa; 651 652 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 653 u16 range_index = nfit_spa->spa->range_index; 654 int type = nfit_spa_type(nfit_spa->spa); 655 struct nfit_memdev *nfit_memdev; 656 657 if (type != NFIT_SPA_BDW) 658 continue; 659 660 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 661 if (nfit_memdev->memdev->range_index != range_index) 662 continue; 663 if (nfit_memdev->memdev->device_handle != device_handle) 664 continue; 665 if (nfit_memdev->memdev->region_index != dcr) 666 continue; 667 668 nfit_mem->spa_bdw = nfit_spa->spa; 669 return; 670 } 671 } 672 673 dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n", 674 nfit_mem->spa_dcr->range_index); 675 nfit_mem->bdw = NULL; 676 } 677 678 static void nfit_mem_init_bdw(struct acpi_nfit_desc *acpi_desc, 679 struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa) 680 { 681 u16 dcr = __to_nfit_memdev(nfit_mem)->region_index; 682 struct nfit_memdev *nfit_memdev; 683 struct nfit_bdw *nfit_bdw; 684 struct nfit_idt *nfit_idt; 685 u16 idt_idx, range_index; 686 687 list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) { 688 if (nfit_bdw->bdw->region_index != dcr) 689 continue; 690 nfit_mem->bdw = nfit_bdw->bdw; 691 break; 692 } 693 694 if (!nfit_mem->bdw) 695 return; 696 697 nfit_mem_find_spa_bdw(acpi_desc, nfit_mem); 698 699 if (!nfit_mem->spa_bdw) 700 return; 701 702 range_index = nfit_mem->spa_bdw->range_index; 703 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 704 if (nfit_memdev->memdev->range_index != range_index || 705 nfit_memdev->memdev->region_index != dcr) 706 continue; 707 nfit_mem->memdev_bdw = nfit_memdev->memdev; 708 idt_idx = nfit_memdev->memdev->interleave_index; 709 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) { 710 if (nfit_idt->idt->interleave_index != idt_idx) 711 continue; 712 nfit_mem->idt_bdw = nfit_idt->idt; 713 break; 714 } 715 break; 716 } 717 } 718 719 static int nfit_mem_dcr_init(struct acpi_nfit_desc *acpi_desc, 720 struct acpi_nfit_system_address *spa) 721 { 722 struct nfit_mem *nfit_mem, *found; 723 struct nfit_memdev *nfit_memdev; 724 int type = nfit_spa_type(spa); 725 726 switch (type) { 727 case NFIT_SPA_DCR: 728 case NFIT_SPA_PM: 729 break; 730 default: 731 return 0; 732 } 733 734 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 735 struct nfit_flush *nfit_flush; 736 struct nfit_dcr *nfit_dcr; 737 u32 device_handle; 738 u16 dcr; 739 740 if (nfit_memdev->memdev->range_index != spa->range_index) 741 continue; 742 found = NULL; 743 dcr = nfit_memdev->memdev->region_index; 744 device_handle = nfit_memdev->memdev->device_handle; 745 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) 746 if (__to_nfit_memdev(nfit_mem)->device_handle 747 == device_handle) { 748 found = nfit_mem; 749 break; 750 } 751 752 if (found) 753 nfit_mem = found; 754 else { 755 nfit_mem = devm_kzalloc(acpi_desc->dev, 756 sizeof(*nfit_mem), GFP_KERNEL); 757 if (!nfit_mem) 758 return -ENOMEM; 759 INIT_LIST_HEAD(&nfit_mem->list); 760 nfit_mem->acpi_desc = acpi_desc; 761 list_add(&nfit_mem->list, &acpi_desc->dimms); 762 } 763 764 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) { 765 if (nfit_dcr->dcr->region_index != dcr) 766 continue; 767 /* 768 * Record the control region for the dimm. For 769 * the ACPI 6.1 case, where there are separate 770 * control regions for the pmem vs blk 771 * interfaces, be sure to record the extended 772 * blk details. 773 */ 774 if (!nfit_mem->dcr) 775 nfit_mem->dcr = nfit_dcr->dcr; 776 else if (nfit_mem->dcr->windows == 0 777 && nfit_dcr->dcr->windows) 778 nfit_mem->dcr = nfit_dcr->dcr; 779 break; 780 } 781 782 list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) { 783 struct acpi_nfit_flush_address *flush; 784 u16 i; 785 786 if (nfit_flush->flush->device_handle != device_handle) 787 continue; 788 nfit_mem->nfit_flush = nfit_flush; 789 flush = nfit_flush->flush; 790 nfit_mem->flush_wpq = devm_kzalloc(acpi_desc->dev, 791 flush->hint_count 792 * sizeof(struct resource), GFP_KERNEL); 793 if (!nfit_mem->flush_wpq) 794 return -ENOMEM; 795 for (i = 0; i < flush->hint_count; i++) { 796 struct resource *res = &nfit_mem->flush_wpq[i]; 797 798 res->start = flush->hint_address[i]; 799 res->end = res->start + 8 - 1; 800 } 801 break; 802 } 803 804 if (dcr && !nfit_mem->dcr) { 805 dev_err(acpi_desc->dev, "SPA %d missing DCR %d\n", 806 spa->range_index, dcr); 807 return -ENODEV; 808 } 809 810 if (type == NFIT_SPA_DCR) { 811 struct nfit_idt *nfit_idt; 812 u16 idt_idx; 813 814 /* multiple dimms may share a SPA when interleaved */ 815 nfit_mem->spa_dcr = spa; 816 nfit_mem->memdev_dcr = nfit_memdev->memdev; 817 idt_idx = nfit_memdev->memdev->interleave_index; 818 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) { 819 if (nfit_idt->idt->interleave_index != idt_idx) 820 continue; 821 nfit_mem->idt_dcr = nfit_idt->idt; 822 break; 823 } 824 nfit_mem_init_bdw(acpi_desc, nfit_mem, spa); 825 } else { 826 /* 827 * A single dimm may belong to multiple SPA-PM 828 * ranges, record at least one in addition to 829 * any SPA-DCR range. 830 */ 831 nfit_mem->memdev_pmem = nfit_memdev->memdev; 832 } 833 } 834 835 return 0; 836 } 837 838 static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b) 839 { 840 struct nfit_mem *a = container_of(_a, typeof(*a), list); 841 struct nfit_mem *b = container_of(_b, typeof(*b), list); 842 u32 handleA, handleB; 843 844 handleA = __to_nfit_memdev(a)->device_handle; 845 handleB = __to_nfit_memdev(b)->device_handle; 846 if (handleA < handleB) 847 return -1; 848 else if (handleA > handleB) 849 return 1; 850 return 0; 851 } 852 853 static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc) 854 { 855 struct nfit_spa *nfit_spa; 856 857 /* 858 * For each SPA-DCR or SPA-PMEM address range find its 859 * corresponding MEMDEV(s). From each MEMDEV find the 860 * corresponding DCR. Then, if we're operating on a SPA-DCR, 861 * try to find a SPA-BDW and a corresponding BDW that references 862 * the DCR. Throw it all into an nfit_mem object. Note, that 863 * BDWs are optional. 864 */ 865 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 866 int rc; 867 868 rc = nfit_mem_dcr_init(acpi_desc, nfit_spa->spa); 869 if (rc) 870 return rc; 871 } 872 873 list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp); 874 875 return 0; 876 } 877 878 static ssize_t revision_show(struct device *dev, 879 struct device_attribute *attr, char *buf) 880 { 881 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 882 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 883 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 884 885 return sprintf(buf, "%d\n", acpi_desc->acpi_header.revision); 886 } 887 static DEVICE_ATTR_RO(revision); 888 889 static ssize_t hw_error_scrub_show(struct device *dev, 890 struct device_attribute *attr, char *buf) 891 { 892 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 893 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 894 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 895 896 return sprintf(buf, "%d\n", acpi_desc->scrub_mode); 897 } 898 899 /* 900 * The 'hw_error_scrub' attribute can have the following values written to it: 901 * '0': Switch to the default mode where an exception will only insert 902 * the address of the memory error into the poison and badblocks lists. 903 * '1': Enable a full scrub to happen if an exception for a memory error is 904 * received. 905 */ 906 static ssize_t hw_error_scrub_store(struct device *dev, 907 struct device_attribute *attr, const char *buf, size_t size) 908 { 909 struct nvdimm_bus_descriptor *nd_desc; 910 ssize_t rc; 911 long val; 912 913 rc = kstrtol(buf, 0, &val); 914 if (rc) 915 return rc; 916 917 device_lock(dev); 918 nd_desc = dev_get_drvdata(dev); 919 if (nd_desc) { 920 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 921 922 switch (val) { 923 case HW_ERROR_SCRUB_ON: 924 acpi_desc->scrub_mode = HW_ERROR_SCRUB_ON; 925 break; 926 case HW_ERROR_SCRUB_OFF: 927 acpi_desc->scrub_mode = HW_ERROR_SCRUB_OFF; 928 break; 929 default: 930 rc = -EINVAL; 931 break; 932 } 933 } 934 device_unlock(dev); 935 if (rc) 936 return rc; 937 return size; 938 } 939 static DEVICE_ATTR_RW(hw_error_scrub); 940 941 /* 942 * This shows the number of full Address Range Scrubs that have been 943 * completed since driver load time. Userspace can wait on this using 944 * select/poll etc. A '+' at the end indicates an ARS is in progress 945 */ 946 static ssize_t scrub_show(struct device *dev, 947 struct device_attribute *attr, char *buf) 948 { 949 struct nvdimm_bus_descriptor *nd_desc; 950 ssize_t rc = -ENXIO; 951 952 device_lock(dev); 953 nd_desc = dev_get_drvdata(dev); 954 if (nd_desc) { 955 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 956 957 rc = sprintf(buf, "%d%s", acpi_desc->scrub_count, 958 (work_busy(&acpi_desc->work)) ? "+\n" : "\n"); 959 } 960 device_unlock(dev); 961 return rc; 962 } 963 964 static ssize_t scrub_store(struct device *dev, 965 struct device_attribute *attr, const char *buf, size_t size) 966 { 967 struct nvdimm_bus_descriptor *nd_desc; 968 ssize_t rc; 969 long val; 970 971 rc = kstrtol(buf, 0, &val); 972 if (rc) 973 return rc; 974 if (val != 1) 975 return -EINVAL; 976 977 device_lock(dev); 978 nd_desc = dev_get_drvdata(dev); 979 if (nd_desc) { 980 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 981 982 rc = acpi_nfit_ars_rescan(acpi_desc); 983 } 984 device_unlock(dev); 985 if (rc) 986 return rc; 987 return size; 988 } 989 static DEVICE_ATTR_RW(scrub); 990 991 static bool ars_supported(struct nvdimm_bus *nvdimm_bus) 992 { 993 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 994 const unsigned long mask = 1 << ND_CMD_ARS_CAP | 1 << ND_CMD_ARS_START 995 | 1 << ND_CMD_ARS_STATUS; 996 997 return (nd_desc->cmd_mask & mask) == mask; 998 } 999 1000 static umode_t nfit_visible(struct kobject *kobj, struct attribute *a, int n) 1001 { 1002 struct device *dev = container_of(kobj, struct device, kobj); 1003 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1004 1005 if (a == &dev_attr_scrub.attr && !ars_supported(nvdimm_bus)) 1006 return 0; 1007 return a->mode; 1008 } 1009 1010 static struct attribute *acpi_nfit_attributes[] = { 1011 &dev_attr_revision.attr, 1012 &dev_attr_scrub.attr, 1013 &dev_attr_hw_error_scrub.attr, 1014 NULL, 1015 }; 1016 1017 static struct attribute_group acpi_nfit_attribute_group = { 1018 .name = "nfit", 1019 .attrs = acpi_nfit_attributes, 1020 .is_visible = nfit_visible, 1021 }; 1022 1023 static const struct attribute_group *acpi_nfit_attribute_groups[] = { 1024 &nvdimm_bus_attribute_group, 1025 &acpi_nfit_attribute_group, 1026 NULL, 1027 }; 1028 1029 static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev) 1030 { 1031 struct nvdimm *nvdimm = to_nvdimm(dev); 1032 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1033 1034 return __to_nfit_memdev(nfit_mem); 1035 } 1036 1037 static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev) 1038 { 1039 struct nvdimm *nvdimm = to_nvdimm(dev); 1040 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1041 1042 return nfit_mem->dcr; 1043 } 1044 1045 static ssize_t handle_show(struct device *dev, 1046 struct device_attribute *attr, char *buf) 1047 { 1048 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev); 1049 1050 return sprintf(buf, "%#x\n", memdev->device_handle); 1051 } 1052 static DEVICE_ATTR_RO(handle); 1053 1054 static ssize_t phys_id_show(struct device *dev, 1055 struct device_attribute *attr, char *buf) 1056 { 1057 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev); 1058 1059 return sprintf(buf, "%#x\n", memdev->physical_id); 1060 } 1061 static DEVICE_ATTR_RO(phys_id); 1062 1063 static ssize_t vendor_show(struct device *dev, 1064 struct device_attribute *attr, char *buf) 1065 { 1066 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1067 1068 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->vendor_id)); 1069 } 1070 static DEVICE_ATTR_RO(vendor); 1071 1072 static ssize_t rev_id_show(struct device *dev, 1073 struct device_attribute *attr, char *buf) 1074 { 1075 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1076 1077 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->revision_id)); 1078 } 1079 static DEVICE_ATTR_RO(rev_id); 1080 1081 static ssize_t device_show(struct device *dev, 1082 struct device_attribute *attr, char *buf) 1083 { 1084 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1085 1086 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->device_id)); 1087 } 1088 static DEVICE_ATTR_RO(device); 1089 1090 static ssize_t subsystem_vendor_show(struct device *dev, 1091 struct device_attribute *attr, char *buf) 1092 { 1093 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1094 1095 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_vendor_id)); 1096 } 1097 static DEVICE_ATTR_RO(subsystem_vendor); 1098 1099 static ssize_t subsystem_rev_id_show(struct device *dev, 1100 struct device_attribute *attr, char *buf) 1101 { 1102 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1103 1104 return sprintf(buf, "0x%04x\n", 1105 be16_to_cpu(dcr->subsystem_revision_id)); 1106 } 1107 static DEVICE_ATTR_RO(subsystem_rev_id); 1108 1109 static ssize_t subsystem_device_show(struct device *dev, 1110 struct device_attribute *attr, char *buf) 1111 { 1112 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1113 1114 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_device_id)); 1115 } 1116 static DEVICE_ATTR_RO(subsystem_device); 1117 1118 static int num_nvdimm_formats(struct nvdimm *nvdimm) 1119 { 1120 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1121 int formats = 0; 1122 1123 if (nfit_mem->memdev_pmem) 1124 formats++; 1125 if (nfit_mem->memdev_bdw) 1126 formats++; 1127 return formats; 1128 } 1129 1130 static ssize_t format_show(struct device *dev, 1131 struct device_attribute *attr, char *buf) 1132 { 1133 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1134 1135 return sprintf(buf, "0x%04x\n", le16_to_cpu(dcr->code)); 1136 } 1137 static DEVICE_ATTR_RO(format); 1138 1139 static ssize_t format1_show(struct device *dev, 1140 struct device_attribute *attr, char *buf) 1141 { 1142 u32 handle; 1143 ssize_t rc = -ENXIO; 1144 struct nfit_mem *nfit_mem; 1145 struct nfit_memdev *nfit_memdev; 1146 struct acpi_nfit_desc *acpi_desc; 1147 struct nvdimm *nvdimm = to_nvdimm(dev); 1148 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1149 1150 nfit_mem = nvdimm_provider_data(nvdimm); 1151 acpi_desc = nfit_mem->acpi_desc; 1152 handle = to_nfit_memdev(dev)->device_handle; 1153 1154 /* assumes DIMMs have at most 2 published interface codes */ 1155 mutex_lock(&acpi_desc->init_mutex); 1156 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 1157 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev; 1158 struct nfit_dcr *nfit_dcr; 1159 1160 if (memdev->device_handle != handle) 1161 continue; 1162 1163 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) { 1164 if (nfit_dcr->dcr->region_index != memdev->region_index) 1165 continue; 1166 if (nfit_dcr->dcr->code == dcr->code) 1167 continue; 1168 rc = sprintf(buf, "0x%04x\n", 1169 le16_to_cpu(nfit_dcr->dcr->code)); 1170 break; 1171 } 1172 if (rc != ENXIO) 1173 break; 1174 } 1175 mutex_unlock(&acpi_desc->init_mutex); 1176 return rc; 1177 } 1178 static DEVICE_ATTR_RO(format1); 1179 1180 static ssize_t formats_show(struct device *dev, 1181 struct device_attribute *attr, char *buf) 1182 { 1183 struct nvdimm *nvdimm = to_nvdimm(dev); 1184 1185 return sprintf(buf, "%d\n", num_nvdimm_formats(nvdimm)); 1186 } 1187 static DEVICE_ATTR_RO(formats); 1188 1189 static ssize_t serial_show(struct device *dev, 1190 struct device_attribute *attr, char *buf) 1191 { 1192 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1193 1194 return sprintf(buf, "0x%08x\n", be32_to_cpu(dcr->serial_number)); 1195 } 1196 static DEVICE_ATTR_RO(serial); 1197 1198 static ssize_t family_show(struct device *dev, 1199 struct device_attribute *attr, char *buf) 1200 { 1201 struct nvdimm *nvdimm = to_nvdimm(dev); 1202 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1203 1204 if (nfit_mem->family < 0) 1205 return -ENXIO; 1206 return sprintf(buf, "%d\n", nfit_mem->family); 1207 } 1208 static DEVICE_ATTR_RO(family); 1209 1210 static ssize_t dsm_mask_show(struct device *dev, 1211 struct device_attribute *attr, char *buf) 1212 { 1213 struct nvdimm *nvdimm = to_nvdimm(dev); 1214 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1215 1216 if (nfit_mem->family < 0) 1217 return -ENXIO; 1218 return sprintf(buf, "%#lx\n", nfit_mem->dsm_mask); 1219 } 1220 static DEVICE_ATTR_RO(dsm_mask); 1221 1222 static ssize_t flags_show(struct device *dev, 1223 struct device_attribute *attr, char *buf) 1224 { 1225 u16 flags = to_nfit_memdev(dev)->flags; 1226 1227 return sprintf(buf, "%s%s%s%s%s\n", 1228 flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "", 1229 flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "", 1230 flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "", 1231 flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "", 1232 flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : ""); 1233 } 1234 static DEVICE_ATTR_RO(flags); 1235 1236 static ssize_t id_show(struct device *dev, 1237 struct device_attribute *attr, char *buf) 1238 { 1239 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1240 1241 if (dcr->valid_fields & ACPI_NFIT_CONTROL_MFG_INFO_VALID) 1242 return sprintf(buf, "%04x-%02x-%04x-%08x\n", 1243 be16_to_cpu(dcr->vendor_id), 1244 dcr->manufacturing_location, 1245 be16_to_cpu(dcr->manufacturing_date), 1246 be32_to_cpu(dcr->serial_number)); 1247 else 1248 return sprintf(buf, "%04x-%08x\n", 1249 be16_to_cpu(dcr->vendor_id), 1250 be32_to_cpu(dcr->serial_number)); 1251 } 1252 static DEVICE_ATTR_RO(id); 1253 1254 static struct attribute *acpi_nfit_dimm_attributes[] = { 1255 &dev_attr_handle.attr, 1256 &dev_attr_phys_id.attr, 1257 &dev_attr_vendor.attr, 1258 &dev_attr_device.attr, 1259 &dev_attr_rev_id.attr, 1260 &dev_attr_subsystem_vendor.attr, 1261 &dev_attr_subsystem_device.attr, 1262 &dev_attr_subsystem_rev_id.attr, 1263 &dev_attr_format.attr, 1264 &dev_attr_formats.attr, 1265 &dev_attr_format1.attr, 1266 &dev_attr_serial.attr, 1267 &dev_attr_flags.attr, 1268 &dev_attr_id.attr, 1269 &dev_attr_family.attr, 1270 &dev_attr_dsm_mask.attr, 1271 NULL, 1272 }; 1273 1274 static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj, 1275 struct attribute *a, int n) 1276 { 1277 struct device *dev = container_of(kobj, struct device, kobj); 1278 struct nvdimm *nvdimm = to_nvdimm(dev); 1279 1280 if (!to_nfit_dcr(dev)) 1281 return 0; 1282 if (a == &dev_attr_format1.attr && num_nvdimm_formats(nvdimm) <= 1) 1283 return 0; 1284 return a->mode; 1285 } 1286 1287 static struct attribute_group acpi_nfit_dimm_attribute_group = { 1288 .name = "nfit", 1289 .attrs = acpi_nfit_dimm_attributes, 1290 .is_visible = acpi_nfit_dimm_attr_visible, 1291 }; 1292 1293 static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = { 1294 &nvdimm_attribute_group, 1295 &nd_device_attribute_group, 1296 &acpi_nfit_dimm_attribute_group, 1297 NULL, 1298 }; 1299 1300 static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc, 1301 u32 device_handle) 1302 { 1303 struct nfit_mem *nfit_mem; 1304 1305 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) 1306 if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle) 1307 return nfit_mem->nvdimm; 1308 1309 return NULL; 1310 } 1311 1312 void __acpi_nvdimm_notify(struct device *dev, u32 event) 1313 { 1314 struct nfit_mem *nfit_mem; 1315 struct acpi_nfit_desc *acpi_desc; 1316 1317 dev_dbg(dev->parent, "%s: %s: event: %d\n", dev_name(dev), __func__, 1318 event); 1319 1320 if (event != NFIT_NOTIFY_DIMM_HEALTH) { 1321 dev_dbg(dev->parent, "%s: unknown event: %d\n", dev_name(dev), 1322 event); 1323 return; 1324 } 1325 1326 acpi_desc = dev_get_drvdata(dev->parent); 1327 if (!acpi_desc) 1328 return; 1329 1330 /* 1331 * If we successfully retrieved acpi_desc, then we know nfit_mem data 1332 * is still valid. 1333 */ 1334 nfit_mem = dev_get_drvdata(dev); 1335 if (nfit_mem && nfit_mem->flags_attr) 1336 sysfs_notify_dirent(nfit_mem->flags_attr); 1337 } 1338 EXPORT_SYMBOL_GPL(__acpi_nvdimm_notify); 1339 1340 static void acpi_nvdimm_notify(acpi_handle handle, u32 event, void *data) 1341 { 1342 struct acpi_device *adev = data; 1343 struct device *dev = &adev->dev; 1344 1345 device_lock(dev->parent); 1346 __acpi_nvdimm_notify(dev, event); 1347 device_unlock(dev->parent); 1348 } 1349 1350 static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc, 1351 struct nfit_mem *nfit_mem, u32 device_handle) 1352 { 1353 struct acpi_device *adev, *adev_dimm; 1354 struct device *dev = acpi_desc->dev; 1355 unsigned long dsm_mask; 1356 const u8 *uuid; 1357 int i; 1358 1359 /* nfit test assumes 1:1 relationship between commands and dsms */ 1360 nfit_mem->dsm_mask = acpi_desc->dimm_cmd_force_en; 1361 nfit_mem->family = NVDIMM_FAMILY_INTEL; 1362 adev = to_acpi_dev(acpi_desc); 1363 if (!adev) 1364 return 0; 1365 1366 adev_dimm = acpi_find_child_device(adev, device_handle, false); 1367 nfit_mem->adev = adev_dimm; 1368 if (!adev_dimm) { 1369 dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n", 1370 device_handle); 1371 return force_enable_dimms ? 0 : -ENODEV; 1372 } 1373 1374 if (ACPI_FAILURE(acpi_install_notify_handler(adev_dimm->handle, 1375 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify, adev_dimm))) { 1376 dev_err(dev, "%s: notification registration failed\n", 1377 dev_name(&adev_dimm->dev)); 1378 return -ENXIO; 1379 } 1380 1381 /* 1382 * Until standardization materializes we need to consider 4 1383 * different command sets. Note, that checking for function0 (bit0) 1384 * tells us if any commands are reachable through this uuid. 1385 */ 1386 for (i = NVDIMM_FAMILY_INTEL; i <= NVDIMM_FAMILY_MSFT; i++) 1387 if (acpi_check_dsm(adev_dimm->handle, to_nfit_uuid(i), 1, 1)) 1388 break; 1389 1390 /* limit the supported commands to those that are publicly documented */ 1391 nfit_mem->family = i; 1392 if (nfit_mem->family == NVDIMM_FAMILY_INTEL) { 1393 dsm_mask = 0x3fe; 1394 if (disable_vendor_specific) 1395 dsm_mask &= ~(1 << ND_CMD_VENDOR); 1396 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE1) { 1397 dsm_mask = 0x1c3c76; 1398 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE2) { 1399 dsm_mask = 0x1fe; 1400 if (disable_vendor_specific) 1401 dsm_mask &= ~(1 << 8); 1402 } else if (nfit_mem->family == NVDIMM_FAMILY_MSFT) { 1403 dsm_mask = 0xffffffff; 1404 } else { 1405 dev_dbg(dev, "unknown dimm command family\n"); 1406 nfit_mem->family = -1; 1407 /* DSMs are optional, continue loading the driver... */ 1408 return 0; 1409 } 1410 1411 uuid = to_nfit_uuid(nfit_mem->family); 1412 for_each_set_bit(i, &dsm_mask, BITS_PER_LONG) 1413 if (acpi_check_dsm(adev_dimm->handle, uuid, 1, 1ULL << i)) 1414 set_bit(i, &nfit_mem->dsm_mask); 1415 1416 return 0; 1417 } 1418 1419 static void shutdown_dimm_notify(void *data) 1420 { 1421 struct acpi_nfit_desc *acpi_desc = data; 1422 struct nfit_mem *nfit_mem; 1423 1424 mutex_lock(&acpi_desc->init_mutex); 1425 /* 1426 * Clear out the nfit_mem->flags_attr and shut down dimm event 1427 * notifications. 1428 */ 1429 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1430 struct acpi_device *adev_dimm = nfit_mem->adev; 1431 1432 if (nfit_mem->flags_attr) { 1433 sysfs_put(nfit_mem->flags_attr); 1434 nfit_mem->flags_attr = NULL; 1435 } 1436 if (adev_dimm) 1437 acpi_remove_notify_handler(adev_dimm->handle, 1438 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify); 1439 } 1440 mutex_unlock(&acpi_desc->init_mutex); 1441 } 1442 1443 static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc) 1444 { 1445 struct nfit_mem *nfit_mem; 1446 int dimm_count = 0, rc; 1447 struct nvdimm *nvdimm; 1448 1449 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1450 struct acpi_nfit_flush_address *flush; 1451 unsigned long flags = 0, cmd_mask; 1452 u32 device_handle; 1453 u16 mem_flags; 1454 1455 device_handle = __to_nfit_memdev(nfit_mem)->device_handle; 1456 nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle); 1457 if (nvdimm) { 1458 dimm_count++; 1459 continue; 1460 } 1461 1462 if (nfit_mem->bdw && nfit_mem->memdev_pmem) 1463 flags |= NDD_ALIASING; 1464 1465 mem_flags = __to_nfit_memdev(nfit_mem)->flags; 1466 if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED) 1467 flags |= NDD_UNARMED; 1468 1469 rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle); 1470 if (rc) 1471 continue; 1472 1473 /* 1474 * TODO: provide translation for non-NVDIMM_FAMILY_INTEL 1475 * devices (i.e. from nd_cmd to acpi_dsm) to standardize the 1476 * userspace interface. 1477 */ 1478 cmd_mask = 1UL << ND_CMD_CALL; 1479 if (nfit_mem->family == NVDIMM_FAMILY_INTEL) 1480 cmd_mask |= nfit_mem->dsm_mask; 1481 1482 flush = nfit_mem->nfit_flush ? nfit_mem->nfit_flush->flush 1483 : NULL; 1484 nvdimm = nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem, 1485 acpi_nfit_dimm_attribute_groups, 1486 flags, cmd_mask, flush ? flush->hint_count : 0, 1487 nfit_mem->flush_wpq); 1488 if (!nvdimm) 1489 return -ENOMEM; 1490 1491 nfit_mem->nvdimm = nvdimm; 1492 dimm_count++; 1493 1494 if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0) 1495 continue; 1496 1497 dev_info(acpi_desc->dev, "%s flags:%s%s%s%s\n", 1498 nvdimm_name(nvdimm), 1499 mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "", 1500 mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"", 1501 mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "", 1502 mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : ""); 1503 1504 } 1505 1506 rc = nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count); 1507 if (rc) 1508 return rc; 1509 1510 /* 1511 * Now that dimms are successfully registered, and async registration 1512 * is flushed, attempt to enable event notification. 1513 */ 1514 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1515 struct kernfs_node *nfit_kernfs; 1516 1517 nvdimm = nfit_mem->nvdimm; 1518 nfit_kernfs = sysfs_get_dirent(nvdimm_kobj(nvdimm)->sd, "nfit"); 1519 if (nfit_kernfs) 1520 nfit_mem->flags_attr = sysfs_get_dirent(nfit_kernfs, 1521 "flags"); 1522 sysfs_put(nfit_kernfs); 1523 if (!nfit_mem->flags_attr) 1524 dev_warn(acpi_desc->dev, "%s: notifications disabled\n", 1525 nvdimm_name(nvdimm)); 1526 } 1527 1528 return devm_add_action_or_reset(acpi_desc->dev, shutdown_dimm_notify, 1529 acpi_desc); 1530 } 1531 1532 static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc) 1533 { 1534 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 1535 const u8 *uuid = to_nfit_uuid(NFIT_DEV_BUS); 1536 struct acpi_device *adev; 1537 int i; 1538 1539 nd_desc->cmd_mask = acpi_desc->bus_cmd_force_en; 1540 adev = to_acpi_dev(acpi_desc); 1541 if (!adev) 1542 return; 1543 1544 for (i = ND_CMD_ARS_CAP; i <= ND_CMD_CLEAR_ERROR; i++) 1545 if (acpi_check_dsm(adev->handle, uuid, 1, 1ULL << i)) 1546 set_bit(i, &nd_desc->cmd_mask); 1547 } 1548 1549 static ssize_t range_index_show(struct device *dev, 1550 struct device_attribute *attr, char *buf) 1551 { 1552 struct nd_region *nd_region = to_nd_region(dev); 1553 struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region); 1554 1555 return sprintf(buf, "%d\n", nfit_spa->spa->range_index); 1556 } 1557 static DEVICE_ATTR_RO(range_index); 1558 1559 static struct attribute *acpi_nfit_region_attributes[] = { 1560 &dev_attr_range_index.attr, 1561 NULL, 1562 }; 1563 1564 static struct attribute_group acpi_nfit_region_attribute_group = { 1565 .name = "nfit", 1566 .attrs = acpi_nfit_region_attributes, 1567 }; 1568 1569 static const struct attribute_group *acpi_nfit_region_attribute_groups[] = { 1570 &nd_region_attribute_group, 1571 &nd_mapping_attribute_group, 1572 &nd_device_attribute_group, 1573 &nd_numa_attribute_group, 1574 &acpi_nfit_region_attribute_group, 1575 NULL, 1576 }; 1577 1578 /* enough info to uniquely specify an interleave set */ 1579 struct nfit_set_info { 1580 struct nfit_set_info_map { 1581 u64 region_offset; 1582 u32 serial_number; 1583 u32 pad; 1584 } mapping[0]; 1585 }; 1586 1587 static size_t sizeof_nfit_set_info(int num_mappings) 1588 { 1589 return sizeof(struct nfit_set_info) 1590 + num_mappings * sizeof(struct nfit_set_info_map); 1591 } 1592 1593 static int cmp_map(const void *m0, const void *m1) 1594 { 1595 const struct nfit_set_info_map *map0 = m0; 1596 const struct nfit_set_info_map *map1 = m1; 1597 1598 return memcmp(&map0->region_offset, &map1->region_offset, 1599 sizeof(u64)); 1600 } 1601 1602 /* Retrieve the nth entry referencing this spa */ 1603 static struct acpi_nfit_memory_map *memdev_from_spa( 1604 struct acpi_nfit_desc *acpi_desc, u16 range_index, int n) 1605 { 1606 struct nfit_memdev *nfit_memdev; 1607 1608 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) 1609 if (nfit_memdev->memdev->range_index == range_index) 1610 if (n-- == 0) 1611 return nfit_memdev->memdev; 1612 return NULL; 1613 } 1614 1615 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc, 1616 struct nd_region_desc *ndr_desc, 1617 struct acpi_nfit_system_address *spa) 1618 { 1619 int i, spa_type = nfit_spa_type(spa); 1620 struct device *dev = acpi_desc->dev; 1621 struct nd_interleave_set *nd_set; 1622 u16 nr = ndr_desc->num_mappings; 1623 struct nfit_set_info *info; 1624 1625 if (spa_type == NFIT_SPA_PM || spa_type == NFIT_SPA_VOLATILE) 1626 /* pass */; 1627 else 1628 return 0; 1629 1630 nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL); 1631 if (!nd_set) 1632 return -ENOMEM; 1633 1634 info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL); 1635 if (!info) 1636 return -ENOMEM; 1637 for (i = 0; i < nr; i++) { 1638 struct nd_mapping_desc *mapping = &ndr_desc->mapping[i]; 1639 struct nfit_set_info_map *map = &info->mapping[i]; 1640 struct nvdimm *nvdimm = mapping->nvdimm; 1641 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1642 struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc, 1643 spa->range_index, i); 1644 1645 if (!memdev || !nfit_mem->dcr) { 1646 dev_err(dev, "%s: failed to find DCR\n", __func__); 1647 return -ENODEV; 1648 } 1649 1650 map->region_offset = memdev->region_offset; 1651 map->serial_number = nfit_mem->dcr->serial_number; 1652 } 1653 1654 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map), 1655 cmp_map, NULL); 1656 nd_set->cookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0); 1657 ndr_desc->nd_set = nd_set; 1658 devm_kfree(dev, info); 1659 1660 return 0; 1661 } 1662 1663 static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio) 1664 { 1665 struct acpi_nfit_interleave *idt = mmio->idt; 1666 u32 sub_line_offset, line_index, line_offset; 1667 u64 line_no, table_skip_count, table_offset; 1668 1669 line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset); 1670 table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index); 1671 line_offset = idt->line_offset[line_index] 1672 * mmio->line_size; 1673 table_offset = table_skip_count * mmio->table_size; 1674 1675 return mmio->base_offset + line_offset + table_offset + sub_line_offset; 1676 } 1677 1678 static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw) 1679 { 1680 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; 1681 u64 offset = nfit_blk->stat_offset + mmio->size * bw; 1682 const u32 STATUS_MASK = 0x80000037; 1683 1684 if (mmio->num_lines) 1685 offset = to_interleave_offset(offset, mmio); 1686 1687 return readl(mmio->addr.base + offset) & STATUS_MASK; 1688 } 1689 1690 static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw, 1691 resource_size_t dpa, unsigned int len, unsigned int write) 1692 { 1693 u64 cmd, offset; 1694 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; 1695 1696 enum { 1697 BCW_OFFSET_MASK = (1ULL << 48)-1, 1698 BCW_LEN_SHIFT = 48, 1699 BCW_LEN_MASK = (1ULL << 8) - 1, 1700 BCW_CMD_SHIFT = 56, 1701 }; 1702 1703 cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK; 1704 len = len >> L1_CACHE_SHIFT; 1705 cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT; 1706 cmd |= ((u64) write) << BCW_CMD_SHIFT; 1707 1708 offset = nfit_blk->cmd_offset + mmio->size * bw; 1709 if (mmio->num_lines) 1710 offset = to_interleave_offset(offset, mmio); 1711 1712 writeq(cmd, mmio->addr.base + offset); 1713 nvdimm_flush(nfit_blk->nd_region); 1714 1715 if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH) 1716 readq(mmio->addr.base + offset); 1717 } 1718 1719 static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk, 1720 resource_size_t dpa, void *iobuf, size_t len, int rw, 1721 unsigned int lane) 1722 { 1723 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW]; 1724 unsigned int copied = 0; 1725 u64 base_offset; 1726 int rc; 1727 1728 base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES 1729 + lane * mmio->size; 1730 write_blk_ctl(nfit_blk, lane, dpa, len, rw); 1731 while (len) { 1732 unsigned int c; 1733 u64 offset; 1734 1735 if (mmio->num_lines) { 1736 u32 line_offset; 1737 1738 offset = to_interleave_offset(base_offset + copied, 1739 mmio); 1740 div_u64_rem(offset, mmio->line_size, &line_offset); 1741 c = min_t(size_t, len, mmio->line_size - line_offset); 1742 } else { 1743 offset = base_offset + nfit_blk->bdw_offset; 1744 c = len; 1745 } 1746 1747 if (rw) 1748 memcpy_to_pmem(mmio->addr.aperture + offset, 1749 iobuf + copied, c); 1750 else { 1751 if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH) 1752 mmio_flush_range((void __force *) 1753 mmio->addr.aperture + offset, c); 1754 1755 memcpy_from_pmem(iobuf + copied, 1756 mmio->addr.aperture + offset, c); 1757 } 1758 1759 copied += c; 1760 len -= c; 1761 } 1762 1763 if (rw) 1764 nvdimm_flush(nfit_blk->nd_region); 1765 1766 rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0; 1767 return rc; 1768 } 1769 1770 static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr, 1771 resource_size_t dpa, void *iobuf, u64 len, int rw) 1772 { 1773 struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr); 1774 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW]; 1775 struct nd_region *nd_region = nfit_blk->nd_region; 1776 unsigned int lane, copied = 0; 1777 int rc = 0; 1778 1779 lane = nd_region_acquire_lane(nd_region); 1780 while (len) { 1781 u64 c = min(len, mmio->size); 1782 1783 rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied, 1784 iobuf + copied, c, rw, lane); 1785 if (rc) 1786 break; 1787 1788 copied += c; 1789 len -= c; 1790 } 1791 nd_region_release_lane(nd_region, lane); 1792 1793 return rc; 1794 } 1795 1796 static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio, 1797 struct acpi_nfit_interleave *idt, u16 interleave_ways) 1798 { 1799 if (idt) { 1800 mmio->num_lines = idt->line_count; 1801 mmio->line_size = idt->line_size; 1802 if (interleave_ways == 0) 1803 return -ENXIO; 1804 mmio->table_size = mmio->num_lines * interleave_ways 1805 * mmio->line_size; 1806 } 1807 1808 return 0; 1809 } 1810 1811 static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc, 1812 struct nvdimm *nvdimm, struct nfit_blk *nfit_blk) 1813 { 1814 struct nd_cmd_dimm_flags flags; 1815 int rc; 1816 1817 memset(&flags, 0, sizeof(flags)); 1818 rc = nd_desc->ndctl(nd_desc, nvdimm, ND_CMD_DIMM_FLAGS, &flags, 1819 sizeof(flags), NULL); 1820 1821 if (rc >= 0 && flags.status == 0) 1822 nfit_blk->dimm_flags = flags.flags; 1823 else if (rc == -ENOTTY) { 1824 /* fall back to a conservative default */ 1825 nfit_blk->dimm_flags = NFIT_BLK_DCR_LATCH | NFIT_BLK_READ_FLUSH; 1826 rc = 0; 1827 } else 1828 rc = -ENXIO; 1829 1830 return rc; 1831 } 1832 1833 static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, 1834 struct device *dev) 1835 { 1836 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1837 struct nd_blk_region *ndbr = to_nd_blk_region(dev); 1838 struct nfit_blk_mmio *mmio; 1839 struct nfit_blk *nfit_blk; 1840 struct nfit_mem *nfit_mem; 1841 struct nvdimm *nvdimm; 1842 int rc; 1843 1844 nvdimm = nd_blk_region_to_dimm(ndbr); 1845 nfit_mem = nvdimm_provider_data(nvdimm); 1846 if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) { 1847 dev_dbg(dev, "%s: missing%s%s%s\n", __func__, 1848 nfit_mem ? "" : " nfit_mem", 1849 (nfit_mem && nfit_mem->dcr) ? "" : " dcr", 1850 (nfit_mem && nfit_mem->bdw) ? "" : " bdw"); 1851 return -ENXIO; 1852 } 1853 1854 nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL); 1855 if (!nfit_blk) 1856 return -ENOMEM; 1857 nd_blk_region_set_provider_data(ndbr, nfit_blk); 1858 nfit_blk->nd_region = to_nd_region(dev); 1859 1860 /* map block aperture memory */ 1861 nfit_blk->bdw_offset = nfit_mem->bdw->offset; 1862 mmio = &nfit_blk->mmio[BDW]; 1863 mmio->addr.base = devm_nvdimm_memremap(dev, nfit_mem->spa_bdw->address, 1864 nfit_mem->spa_bdw->length, ARCH_MEMREMAP_PMEM); 1865 if (!mmio->addr.base) { 1866 dev_dbg(dev, "%s: %s failed to map bdw\n", __func__, 1867 nvdimm_name(nvdimm)); 1868 return -ENOMEM; 1869 } 1870 mmio->size = nfit_mem->bdw->size; 1871 mmio->base_offset = nfit_mem->memdev_bdw->region_offset; 1872 mmio->idt = nfit_mem->idt_bdw; 1873 mmio->spa = nfit_mem->spa_bdw; 1874 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw, 1875 nfit_mem->memdev_bdw->interleave_ways); 1876 if (rc) { 1877 dev_dbg(dev, "%s: %s failed to init bdw interleave\n", 1878 __func__, nvdimm_name(nvdimm)); 1879 return rc; 1880 } 1881 1882 /* map block control memory */ 1883 nfit_blk->cmd_offset = nfit_mem->dcr->command_offset; 1884 nfit_blk->stat_offset = nfit_mem->dcr->status_offset; 1885 mmio = &nfit_blk->mmio[DCR]; 1886 mmio->addr.base = devm_nvdimm_ioremap(dev, nfit_mem->spa_dcr->address, 1887 nfit_mem->spa_dcr->length); 1888 if (!mmio->addr.base) { 1889 dev_dbg(dev, "%s: %s failed to map dcr\n", __func__, 1890 nvdimm_name(nvdimm)); 1891 return -ENOMEM; 1892 } 1893 mmio->size = nfit_mem->dcr->window_size; 1894 mmio->base_offset = nfit_mem->memdev_dcr->region_offset; 1895 mmio->idt = nfit_mem->idt_dcr; 1896 mmio->spa = nfit_mem->spa_dcr; 1897 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr, 1898 nfit_mem->memdev_dcr->interleave_ways); 1899 if (rc) { 1900 dev_dbg(dev, "%s: %s failed to init dcr interleave\n", 1901 __func__, nvdimm_name(nvdimm)); 1902 return rc; 1903 } 1904 1905 rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk); 1906 if (rc < 0) { 1907 dev_dbg(dev, "%s: %s failed get DIMM flags\n", 1908 __func__, nvdimm_name(nvdimm)); 1909 return rc; 1910 } 1911 1912 if (nvdimm_has_flush(nfit_blk->nd_region) < 0) 1913 dev_warn(dev, "unable to guarantee persistence of writes\n"); 1914 1915 if (mmio->line_size == 0) 1916 return 0; 1917 1918 if ((u32) nfit_blk->cmd_offset % mmio->line_size 1919 + 8 > mmio->line_size) { 1920 dev_dbg(dev, "cmd_offset crosses interleave boundary\n"); 1921 return -ENXIO; 1922 } else if ((u32) nfit_blk->stat_offset % mmio->line_size 1923 + 8 > mmio->line_size) { 1924 dev_dbg(dev, "stat_offset crosses interleave boundary\n"); 1925 return -ENXIO; 1926 } 1927 1928 return 0; 1929 } 1930 1931 static int ars_get_cap(struct acpi_nfit_desc *acpi_desc, 1932 struct nd_cmd_ars_cap *cmd, struct nfit_spa *nfit_spa) 1933 { 1934 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 1935 struct acpi_nfit_system_address *spa = nfit_spa->spa; 1936 int cmd_rc, rc; 1937 1938 cmd->address = spa->address; 1939 cmd->length = spa->length; 1940 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, cmd, 1941 sizeof(*cmd), &cmd_rc); 1942 if (rc < 0) 1943 return rc; 1944 return cmd_rc; 1945 } 1946 1947 static int ars_start(struct acpi_nfit_desc *acpi_desc, struct nfit_spa *nfit_spa) 1948 { 1949 int rc; 1950 int cmd_rc; 1951 struct nd_cmd_ars_start ars_start; 1952 struct acpi_nfit_system_address *spa = nfit_spa->spa; 1953 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 1954 1955 memset(&ars_start, 0, sizeof(ars_start)); 1956 ars_start.address = spa->address; 1957 ars_start.length = spa->length; 1958 if (nfit_spa_type(spa) == NFIT_SPA_PM) 1959 ars_start.type = ND_ARS_PERSISTENT; 1960 else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE) 1961 ars_start.type = ND_ARS_VOLATILE; 1962 else 1963 return -ENOTTY; 1964 1965 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start, 1966 sizeof(ars_start), &cmd_rc); 1967 1968 if (rc < 0) 1969 return rc; 1970 return cmd_rc; 1971 } 1972 1973 static int ars_continue(struct acpi_nfit_desc *acpi_desc) 1974 { 1975 int rc, cmd_rc; 1976 struct nd_cmd_ars_start ars_start; 1977 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 1978 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; 1979 1980 memset(&ars_start, 0, sizeof(ars_start)); 1981 ars_start.address = ars_status->restart_address; 1982 ars_start.length = ars_status->restart_length; 1983 ars_start.type = ars_status->type; 1984 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start, 1985 sizeof(ars_start), &cmd_rc); 1986 if (rc < 0) 1987 return rc; 1988 return cmd_rc; 1989 } 1990 1991 static int ars_get_status(struct acpi_nfit_desc *acpi_desc) 1992 { 1993 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 1994 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; 1995 int rc, cmd_rc; 1996 1997 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_STATUS, ars_status, 1998 acpi_desc->ars_status_size, &cmd_rc); 1999 if (rc < 0) 2000 return rc; 2001 return cmd_rc; 2002 } 2003 2004 static int ars_status_process_records(struct nvdimm_bus *nvdimm_bus, 2005 struct nd_cmd_ars_status *ars_status) 2006 { 2007 int rc; 2008 u32 i; 2009 2010 for (i = 0; i < ars_status->num_records; i++) { 2011 rc = nvdimm_bus_add_poison(nvdimm_bus, 2012 ars_status->records[i].err_address, 2013 ars_status->records[i].length); 2014 if (rc) 2015 return rc; 2016 } 2017 2018 return 0; 2019 } 2020 2021 static void acpi_nfit_remove_resource(void *data) 2022 { 2023 struct resource *res = data; 2024 2025 remove_resource(res); 2026 } 2027 2028 static int acpi_nfit_insert_resource(struct acpi_nfit_desc *acpi_desc, 2029 struct nd_region_desc *ndr_desc) 2030 { 2031 struct resource *res, *nd_res = ndr_desc->res; 2032 int is_pmem, ret; 2033 2034 /* No operation if the region is already registered as PMEM */ 2035 is_pmem = region_intersects(nd_res->start, resource_size(nd_res), 2036 IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY); 2037 if (is_pmem == REGION_INTERSECTS) 2038 return 0; 2039 2040 res = devm_kzalloc(acpi_desc->dev, sizeof(*res), GFP_KERNEL); 2041 if (!res) 2042 return -ENOMEM; 2043 2044 res->name = "Persistent Memory"; 2045 res->start = nd_res->start; 2046 res->end = nd_res->end; 2047 res->flags = IORESOURCE_MEM; 2048 res->desc = IORES_DESC_PERSISTENT_MEMORY; 2049 2050 ret = insert_resource(&iomem_resource, res); 2051 if (ret) 2052 return ret; 2053 2054 ret = devm_add_action_or_reset(acpi_desc->dev, 2055 acpi_nfit_remove_resource, 2056 res); 2057 if (ret) 2058 return ret; 2059 2060 return 0; 2061 } 2062 2063 static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc, 2064 struct nd_mapping_desc *mapping, struct nd_region_desc *ndr_desc, 2065 struct acpi_nfit_memory_map *memdev, 2066 struct nfit_spa *nfit_spa) 2067 { 2068 struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, 2069 memdev->device_handle); 2070 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2071 struct nd_blk_region_desc *ndbr_desc; 2072 struct nfit_mem *nfit_mem; 2073 int blk_valid = 0; 2074 2075 if (!nvdimm) { 2076 dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n", 2077 spa->range_index, memdev->device_handle); 2078 return -ENODEV; 2079 } 2080 2081 mapping->nvdimm = nvdimm; 2082 switch (nfit_spa_type(spa)) { 2083 case NFIT_SPA_PM: 2084 case NFIT_SPA_VOLATILE: 2085 mapping->start = memdev->address; 2086 mapping->size = memdev->region_size; 2087 break; 2088 case NFIT_SPA_DCR: 2089 nfit_mem = nvdimm_provider_data(nvdimm); 2090 if (!nfit_mem || !nfit_mem->bdw) { 2091 dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n", 2092 spa->range_index, nvdimm_name(nvdimm)); 2093 } else { 2094 mapping->size = nfit_mem->bdw->capacity; 2095 mapping->start = nfit_mem->bdw->start_address; 2096 ndr_desc->num_lanes = nfit_mem->bdw->windows; 2097 blk_valid = 1; 2098 } 2099 2100 ndr_desc->mapping = mapping; 2101 ndr_desc->num_mappings = blk_valid; 2102 ndbr_desc = to_blk_region_desc(ndr_desc); 2103 ndbr_desc->enable = acpi_nfit_blk_region_enable; 2104 ndbr_desc->do_io = acpi_desc->blk_do_io; 2105 nfit_spa->nd_region = nvdimm_blk_region_create(acpi_desc->nvdimm_bus, 2106 ndr_desc); 2107 if (!nfit_spa->nd_region) 2108 return -ENOMEM; 2109 break; 2110 } 2111 2112 return 0; 2113 } 2114 2115 static bool nfit_spa_is_virtual(struct acpi_nfit_system_address *spa) 2116 { 2117 return (nfit_spa_type(spa) == NFIT_SPA_VDISK || 2118 nfit_spa_type(spa) == NFIT_SPA_VCD || 2119 nfit_spa_type(spa) == NFIT_SPA_PDISK || 2120 nfit_spa_type(spa) == NFIT_SPA_PCD); 2121 } 2122 2123 static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc, 2124 struct nfit_spa *nfit_spa) 2125 { 2126 static struct nd_mapping_desc mappings[ND_MAX_MAPPINGS]; 2127 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2128 struct nd_blk_region_desc ndbr_desc; 2129 struct nd_region_desc *ndr_desc; 2130 struct nfit_memdev *nfit_memdev; 2131 struct nvdimm_bus *nvdimm_bus; 2132 struct resource res; 2133 int count = 0, rc; 2134 2135 if (nfit_spa->nd_region) 2136 return 0; 2137 2138 if (spa->range_index == 0 && !nfit_spa_is_virtual(spa)) { 2139 dev_dbg(acpi_desc->dev, "%s: detected invalid spa index\n", 2140 __func__); 2141 return 0; 2142 } 2143 2144 memset(&res, 0, sizeof(res)); 2145 memset(&mappings, 0, sizeof(mappings)); 2146 memset(&ndbr_desc, 0, sizeof(ndbr_desc)); 2147 res.start = spa->address; 2148 res.end = res.start + spa->length - 1; 2149 ndr_desc = &ndbr_desc.ndr_desc; 2150 ndr_desc->res = &res; 2151 ndr_desc->provider_data = nfit_spa; 2152 ndr_desc->attr_groups = acpi_nfit_region_attribute_groups; 2153 if (spa->flags & ACPI_NFIT_PROXIMITY_VALID) 2154 ndr_desc->numa_node = acpi_map_pxm_to_online_node( 2155 spa->proximity_domain); 2156 else 2157 ndr_desc->numa_node = NUMA_NO_NODE; 2158 2159 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 2160 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev; 2161 struct nd_mapping_desc *mapping; 2162 2163 if (memdev->range_index != spa->range_index) 2164 continue; 2165 if (count >= ND_MAX_MAPPINGS) { 2166 dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n", 2167 spa->range_index, ND_MAX_MAPPINGS); 2168 return -ENXIO; 2169 } 2170 mapping = &mappings[count++]; 2171 rc = acpi_nfit_init_mapping(acpi_desc, mapping, ndr_desc, 2172 memdev, nfit_spa); 2173 if (rc) 2174 goto out; 2175 } 2176 2177 ndr_desc->mapping = mappings; 2178 ndr_desc->num_mappings = count; 2179 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa); 2180 if (rc) 2181 goto out; 2182 2183 nvdimm_bus = acpi_desc->nvdimm_bus; 2184 if (nfit_spa_type(spa) == NFIT_SPA_PM) { 2185 rc = acpi_nfit_insert_resource(acpi_desc, ndr_desc); 2186 if (rc) { 2187 dev_warn(acpi_desc->dev, 2188 "failed to insert pmem resource to iomem: %d\n", 2189 rc); 2190 goto out; 2191 } 2192 2193 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus, 2194 ndr_desc); 2195 if (!nfit_spa->nd_region) 2196 rc = -ENOMEM; 2197 } else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE) { 2198 nfit_spa->nd_region = nvdimm_volatile_region_create(nvdimm_bus, 2199 ndr_desc); 2200 if (!nfit_spa->nd_region) 2201 rc = -ENOMEM; 2202 } else if (nfit_spa_is_virtual(spa)) { 2203 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus, 2204 ndr_desc); 2205 if (!nfit_spa->nd_region) 2206 rc = -ENOMEM; 2207 } 2208 2209 out: 2210 if (rc) 2211 dev_err(acpi_desc->dev, "failed to register spa range %d\n", 2212 nfit_spa->spa->range_index); 2213 return rc; 2214 } 2215 2216 static int ars_status_alloc(struct acpi_nfit_desc *acpi_desc, 2217 u32 max_ars) 2218 { 2219 struct device *dev = acpi_desc->dev; 2220 struct nd_cmd_ars_status *ars_status; 2221 2222 if (acpi_desc->ars_status && acpi_desc->ars_status_size >= max_ars) { 2223 memset(acpi_desc->ars_status, 0, acpi_desc->ars_status_size); 2224 return 0; 2225 } 2226 2227 if (acpi_desc->ars_status) 2228 devm_kfree(dev, acpi_desc->ars_status); 2229 acpi_desc->ars_status = NULL; 2230 ars_status = devm_kzalloc(dev, max_ars, GFP_KERNEL); 2231 if (!ars_status) 2232 return -ENOMEM; 2233 acpi_desc->ars_status = ars_status; 2234 acpi_desc->ars_status_size = max_ars; 2235 return 0; 2236 } 2237 2238 static int acpi_nfit_query_poison(struct acpi_nfit_desc *acpi_desc, 2239 struct nfit_spa *nfit_spa) 2240 { 2241 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2242 int rc; 2243 2244 if (!nfit_spa->max_ars) { 2245 struct nd_cmd_ars_cap ars_cap; 2246 2247 memset(&ars_cap, 0, sizeof(ars_cap)); 2248 rc = ars_get_cap(acpi_desc, &ars_cap, nfit_spa); 2249 if (rc < 0) 2250 return rc; 2251 nfit_spa->max_ars = ars_cap.max_ars_out; 2252 nfit_spa->clear_err_unit = ars_cap.clear_err_unit; 2253 /* check that the supported scrub types match the spa type */ 2254 if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE && 2255 ((ars_cap.status >> 16) & ND_ARS_VOLATILE) == 0) 2256 return -ENOTTY; 2257 else if (nfit_spa_type(spa) == NFIT_SPA_PM && 2258 ((ars_cap.status >> 16) & ND_ARS_PERSISTENT) == 0) 2259 return -ENOTTY; 2260 } 2261 2262 if (ars_status_alloc(acpi_desc, nfit_spa->max_ars)) 2263 return -ENOMEM; 2264 2265 rc = ars_get_status(acpi_desc); 2266 if (rc < 0 && rc != -ENOSPC) 2267 return rc; 2268 2269 if (ars_status_process_records(acpi_desc->nvdimm_bus, 2270 acpi_desc->ars_status)) 2271 return -ENOMEM; 2272 2273 return 0; 2274 } 2275 2276 static void acpi_nfit_async_scrub(struct acpi_nfit_desc *acpi_desc, 2277 struct nfit_spa *nfit_spa) 2278 { 2279 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2280 unsigned int overflow_retry = scrub_overflow_abort; 2281 u64 init_ars_start = 0, init_ars_len = 0; 2282 struct device *dev = acpi_desc->dev; 2283 unsigned int tmo = scrub_timeout; 2284 int rc; 2285 2286 if (!nfit_spa->ars_required || !nfit_spa->nd_region) 2287 return; 2288 2289 rc = ars_start(acpi_desc, nfit_spa); 2290 /* 2291 * If we timed out the initial scan we'll still be busy here, 2292 * and will wait another timeout before giving up permanently. 2293 */ 2294 if (rc < 0 && rc != -EBUSY) 2295 return; 2296 2297 do { 2298 u64 ars_start, ars_len; 2299 2300 if (acpi_desc->cancel) 2301 break; 2302 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa); 2303 if (rc == -ENOTTY) 2304 break; 2305 if (rc == -EBUSY && !tmo) { 2306 dev_warn(dev, "range %d ars timeout, aborting\n", 2307 spa->range_index); 2308 break; 2309 } 2310 2311 if (rc == -EBUSY) { 2312 /* 2313 * Note, entries may be appended to the list 2314 * while the lock is dropped, but the workqueue 2315 * being active prevents entries being deleted / 2316 * freed. 2317 */ 2318 mutex_unlock(&acpi_desc->init_mutex); 2319 ssleep(1); 2320 tmo--; 2321 mutex_lock(&acpi_desc->init_mutex); 2322 continue; 2323 } 2324 2325 /* we got some results, but there are more pending... */ 2326 if (rc == -ENOSPC && overflow_retry--) { 2327 if (!init_ars_len) { 2328 init_ars_len = acpi_desc->ars_status->length; 2329 init_ars_start = acpi_desc->ars_status->address; 2330 } 2331 rc = ars_continue(acpi_desc); 2332 } 2333 2334 if (rc < 0) { 2335 dev_warn(dev, "range %d ars continuation failed\n", 2336 spa->range_index); 2337 break; 2338 } 2339 2340 if (init_ars_len) { 2341 ars_start = init_ars_start; 2342 ars_len = init_ars_len; 2343 } else { 2344 ars_start = acpi_desc->ars_status->address; 2345 ars_len = acpi_desc->ars_status->length; 2346 } 2347 dev_dbg(dev, "spa range: %d ars from %#llx + %#llx complete\n", 2348 spa->range_index, ars_start, ars_len); 2349 /* notify the region about new poison entries */ 2350 nvdimm_region_notify(nfit_spa->nd_region, 2351 NVDIMM_REVALIDATE_POISON); 2352 break; 2353 } while (1); 2354 } 2355 2356 static void acpi_nfit_scrub(struct work_struct *work) 2357 { 2358 struct device *dev; 2359 u64 init_scrub_length = 0; 2360 struct nfit_spa *nfit_spa; 2361 u64 init_scrub_address = 0; 2362 bool init_ars_done = false; 2363 struct acpi_nfit_desc *acpi_desc; 2364 unsigned int tmo = scrub_timeout; 2365 unsigned int overflow_retry = scrub_overflow_abort; 2366 2367 acpi_desc = container_of(work, typeof(*acpi_desc), work); 2368 dev = acpi_desc->dev; 2369 2370 /* 2371 * We scrub in 2 phases. The first phase waits for any platform 2372 * firmware initiated scrubs to complete and then we go search for the 2373 * affected spa regions to mark them scanned. In the second phase we 2374 * initiate a directed scrub for every range that was not scrubbed in 2375 * phase 1. If we're called for a 'rescan', we harmlessly pass through 2376 * the first phase, but really only care about running phase 2, where 2377 * regions can be notified of new poison. 2378 */ 2379 2380 /* process platform firmware initiated scrubs */ 2381 retry: 2382 mutex_lock(&acpi_desc->init_mutex); 2383 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 2384 struct nd_cmd_ars_status *ars_status; 2385 struct acpi_nfit_system_address *spa; 2386 u64 ars_start, ars_len; 2387 int rc; 2388 2389 if (acpi_desc->cancel) 2390 break; 2391 2392 if (nfit_spa->nd_region) 2393 continue; 2394 2395 if (init_ars_done) { 2396 /* 2397 * No need to re-query, we're now just 2398 * reconciling all the ranges covered by the 2399 * initial scrub 2400 */ 2401 rc = 0; 2402 } else 2403 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa); 2404 2405 if (rc == -ENOTTY) { 2406 /* no ars capability, just register spa and move on */ 2407 acpi_nfit_register_region(acpi_desc, nfit_spa); 2408 continue; 2409 } 2410 2411 if (rc == -EBUSY && !tmo) { 2412 /* fallthrough to directed scrub in phase 2 */ 2413 dev_warn(dev, "timeout awaiting ars results, continuing...\n"); 2414 break; 2415 } else if (rc == -EBUSY) { 2416 mutex_unlock(&acpi_desc->init_mutex); 2417 ssleep(1); 2418 tmo--; 2419 goto retry; 2420 } 2421 2422 /* we got some results, but there are more pending... */ 2423 if (rc == -ENOSPC && overflow_retry--) { 2424 ars_status = acpi_desc->ars_status; 2425 /* 2426 * Record the original scrub range, so that we 2427 * can recall all the ranges impacted by the 2428 * initial scrub. 2429 */ 2430 if (!init_scrub_length) { 2431 init_scrub_length = ars_status->length; 2432 init_scrub_address = ars_status->address; 2433 } 2434 rc = ars_continue(acpi_desc); 2435 if (rc == 0) { 2436 mutex_unlock(&acpi_desc->init_mutex); 2437 goto retry; 2438 } 2439 } 2440 2441 if (rc < 0) { 2442 /* 2443 * Initial scrub failed, we'll give it one more 2444 * try below... 2445 */ 2446 break; 2447 } 2448 2449 /* We got some final results, record completed ranges */ 2450 ars_status = acpi_desc->ars_status; 2451 if (init_scrub_length) { 2452 ars_start = init_scrub_address; 2453 ars_len = ars_start + init_scrub_length; 2454 } else { 2455 ars_start = ars_status->address; 2456 ars_len = ars_status->length; 2457 } 2458 spa = nfit_spa->spa; 2459 2460 if (!init_ars_done) { 2461 init_ars_done = true; 2462 dev_dbg(dev, "init scrub %#llx + %#llx complete\n", 2463 ars_start, ars_len); 2464 } 2465 if (ars_start <= spa->address && ars_start + ars_len 2466 >= spa->address + spa->length) 2467 acpi_nfit_register_region(acpi_desc, nfit_spa); 2468 } 2469 2470 /* 2471 * For all the ranges not covered by an initial scrub we still 2472 * want to see if there are errors, but it's ok to discover them 2473 * asynchronously. 2474 */ 2475 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 2476 /* 2477 * Flag all the ranges that still need scrubbing, but 2478 * register them now to make data available. 2479 */ 2480 if (!nfit_spa->nd_region) { 2481 nfit_spa->ars_required = 1; 2482 acpi_nfit_register_region(acpi_desc, nfit_spa); 2483 } 2484 } 2485 2486 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) 2487 acpi_nfit_async_scrub(acpi_desc, nfit_spa); 2488 acpi_desc->scrub_count++; 2489 if (acpi_desc->scrub_count_state) 2490 sysfs_notify_dirent(acpi_desc->scrub_count_state); 2491 mutex_unlock(&acpi_desc->init_mutex); 2492 } 2493 2494 static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc) 2495 { 2496 struct nfit_spa *nfit_spa; 2497 int rc; 2498 2499 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) 2500 if (nfit_spa_type(nfit_spa->spa) == NFIT_SPA_DCR) { 2501 /* BLK regions don't need to wait for ars results */ 2502 rc = acpi_nfit_register_region(acpi_desc, nfit_spa); 2503 if (rc) 2504 return rc; 2505 } 2506 2507 queue_work(nfit_wq, &acpi_desc->work); 2508 return 0; 2509 } 2510 2511 static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc, 2512 struct nfit_table_prev *prev) 2513 { 2514 struct device *dev = acpi_desc->dev; 2515 2516 if (!list_empty(&prev->spas) || 2517 !list_empty(&prev->memdevs) || 2518 !list_empty(&prev->dcrs) || 2519 !list_empty(&prev->bdws) || 2520 !list_empty(&prev->idts) || 2521 !list_empty(&prev->flushes)) { 2522 dev_err(dev, "new nfit deletes entries (unsupported)\n"); 2523 return -ENXIO; 2524 } 2525 return 0; 2526 } 2527 2528 static int acpi_nfit_desc_init_scrub_attr(struct acpi_nfit_desc *acpi_desc) 2529 { 2530 struct device *dev = acpi_desc->dev; 2531 struct kernfs_node *nfit; 2532 struct device *bus_dev; 2533 2534 if (!ars_supported(acpi_desc->nvdimm_bus)) 2535 return 0; 2536 2537 bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); 2538 nfit = sysfs_get_dirent(bus_dev->kobj.sd, "nfit"); 2539 if (!nfit) { 2540 dev_err(dev, "sysfs_get_dirent 'nfit' failed\n"); 2541 return -ENODEV; 2542 } 2543 acpi_desc->scrub_count_state = sysfs_get_dirent(nfit, "scrub"); 2544 sysfs_put(nfit); 2545 if (!acpi_desc->scrub_count_state) { 2546 dev_err(dev, "sysfs_get_dirent 'scrub' failed\n"); 2547 return -ENODEV; 2548 } 2549 2550 return 0; 2551 } 2552 2553 static void acpi_nfit_destruct(void *data) 2554 { 2555 struct acpi_nfit_desc *acpi_desc = data; 2556 struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); 2557 2558 /* 2559 * Destruct under acpi_desc_lock so that nfit_handle_mce does not 2560 * race teardown 2561 */ 2562 mutex_lock(&acpi_desc_lock); 2563 acpi_desc->cancel = 1; 2564 /* 2565 * Bounce the nvdimm bus lock to make sure any in-flight 2566 * acpi_nfit_ars_rescan() submissions have had a chance to 2567 * either submit or see ->cancel set. 2568 */ 2569 device_lock(bus_dev); 2570 device_unlock(bus_dev); 2571 2572 flush_workqueue(nfit_wq); 2573 if (acpi_desc->scrub_count_state) 2574 sysfs_put(acpi_desc->scrub_count_state); 2575 nvdimm_bus_unregister(acpi_desc->nvdimm_bus); 2576 acpi_desc->nvdimm_bus = NULL; 2577 list_del(&acpi_desc->list); 2578 mutex_unlock(&acpi_desc_lock); 2579 } 2580 2581 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz) 2582 { 2583 struct device *dev = acpi_desc->dev; 2584 struct nfit_table_prev prev; 2585 const void *end; 2586 int rc; 2587 2588 if (!acpi_desc->nvdimm_bus) { 2589 acpi_nfit_init_dsms(acpi_desc); 2590 2591 acpi_desc->nvdimm_bus = nvdimm_bus_register(dev, 2592 &acpi_desc->nd_desc); 2593 if (!acpi_desc->nvdimm_bus) 2594 return -ENOMEM; 2595 2596 rc = devm_add_action_or_reset(dev, acpi_nfit_destruct, 2597 acpi_desc); 2598 if (rc) 2599 return rc; 2600 2601 rc = acpi_nfit_desc_init_scrub_attr(acpi_desc); 2602 if (rc) 2603 return rc; 2604 2605 /* register this acpi_desc for mce notifications */ 2606 mutex_lock(&acpi_desc_lock); 2607 list_add_tail(&acpi_desc->list, &acpi_descs); 2608 mutex_unlock(&acpi_desc_lock); 2609 } 2610 2611 mutex_lock(&acpi_desc->init_mutex); 2612 2613 INIT_LIST_HEAD(&prev.spas); 2614 INIT_LIST_HEAD(&prev.memdevs); 2615 INIT_LIST_HEAD(&prev.dcrs); 2616 INIT_LIST_HEAD(&prev.bdws); 2617 INIT_LIST_HEAD(&prev.idts); 2618 INIT_LIST_HEAD(&prev.flushes); 2619 2620 list_cut_position(&prev.spas, &acpi_desc->spas, 2621 acpi_desc->spas.prev); 2622 list_cut_position(&prev.memdevs, &acpi_desc->memdevs, 2623 acpi_desc->memdevs.prev); 2624 list_cut_position(&prev.dcrs, &acpi_desc->dcrs, 2625 acpi_desc->dcrs.prev); 2626 list_cut_position(&prev.bdws, &acpi_desc->bdws, 2627 acpi_desc->bdws.prev); 2628 list_cut_position(&prev.idts, &acpi_desc->idts, 2629 acpi_desc->idts.prev); 2630 list_cut_position(&prev.flushes, &acpi_desc->flushes, 2631 acpi_desc->flushes.prev); 2632 2633 end = data + sz; 2634 while (!IS_ERR_OR_NULL(data)) 2635 data = add_table(acpi_desc, &prev, data, end); 2636 2637 if (IS_ERR(data)) { 2638 dev_dbg(dev, "%s: nfit table parsing error: %ld\n", __func__, 2639 PTR_ERR(data)); 2640 rc = PTR_ERR(data); 2641 goto out_unlock; 2642 } 2643 2644 rc = acpi_nfit_check_deletions(acpi_desc, &prev); 2645 if (rc) 2646 goto out_unlock; 2647 2648 rc = nfit_mem_init(acpi_desc); 2649 if (rc) 2650 goto out_unlock; 2651 2652 rc = acpi_nfit_register_dimms(acpi_desc); 2653 if (rc) 2654 goto out_unlock; 2655 2656 rc = acpi_nfit_register_regions(acpi_desc); 2657 2658 out_unlock: 2659 mutex_unlock(&acpi_desc->init_mutex); 2660 return rc; 2661 } 2662 EXPORT_SYMBOL_GPL(acpi_nfit_init); 2663 2664 struct acpi_nfit_flush_work { 2665 struct work_struct work; 2666 struct completion cmp; 2667 }; 2668 2669 static void flush_probe(struct work_struct *work) 2670 { 2671 struct acpi_nfit_flush_work *flush; 2672 2673 flush = container_of(work, typeof(*flush), work); 2674 complete(&flush->cmp); 2675 } 2676 2677 static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc) 2678 { 2679 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 2680 struct device *dev = acpi_desc->dev; 2681 struct acpi_nfit_flush_work flush; 2682 2683 /* bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */ 2684 device_lock(dev); 2685 device_unlock(dev); 2686 2687 /* 2688 * Scrub work could take 10s of seconds, userspace may give up so we 2689 * need to be interruptible while waiting. 2690 */ 2691 INIT_WORK_ONSTACK(&flush.work, flush_probe); 2692 COMPLETION_INITIALIZER_ONSTACK(flush.cmp); 2693 queue_work(nfit_wq, &flush.work); 2694 return wait_for_completion_interruptible(&flush.cmp); 2695 } 2696 2697 static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc, 2698 struct nvdimm *nvdimm, unsigned int cmd) 2699 { 2700 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 2701 2702 if (nvdimm) 2703 return 0; 2704 if (cmd != ND_CMD_ARS_START) 2705 return 0; 2706 2707 /* 2708 * The kernel and userspace may race to initiate a scrub, but 2709 * the scrub thread is prepared to lose that initial race. It 2710 * just needs guarantees that any ars it initiates are not 2711 * interrupted by any intervening start reqeusts from userspace. 2712 */ 2713 if (work_busy(&acpi_desc->work)) 2714 return -EBUSY; 2715 2716 return 0; 2717 } 2718 2719 int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc) 2720 { 2721 struct device *dev = acpi_desc->dev; 2722 struct nfit_spa *nfit_spa; 2723 2724 if (work_busy(&acpi_desc->work)) 2725 return -EBUSY; 2726 2727 if (acpi_desc->cancel) 2728 return 0; 2729 2730 mutex_lock(&acpi_desc->init_mutex); 2731 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 2732 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2733 2734 if (nfit_spa_type(spa) != NFIT_SPA_PM) 2735 continue; 2736 2737 nfit_spa->ars_required = 1; 2738 } 2739 queue_work(nfit_wq, &acpi_desc->work); 2740 dev_dbg(dev, "%s: ars_scan triggered\n", __func__); 2741 mutex_unlock(&acpi_desc->init_mutex); 2742 2743 return 0; 2744 } 2745 2746 void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev) 2747 { 2748 struct nvdimm_bus_descriptor *nd_desc; 2749 2750 dev_set_drvdata(dev, acpi_desc); 2751 acpi_desc->dev = dev; 2752 acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io; 2753 nd_desc = &acpi_desc->nd_desc; 2754 nd_desc->provider_name = "ACPI.NFIT"; 2755 nd_desc->module = THIS_MODULE; 2756 nd_desc->ndctl = acpi_nfit_ctl; 2757 nd_desc->flush_probe = acpi_nfit_flush_probe; 2758 nd_desc->clear_to_send = acpi_nfit_clear_to_send; 2759 nd_desc->attr_groups = acpi_nfit_attribute_groups; 2760 2761 INIT_LIST_HEAD(&acpi_desc->spas); 2762 INIT_LIST_HEAD(&acpi_desc->dcrs); 2763 INIT_LIST_HEAD(&acpi_desc->bdws); 2764 INIT_LIST_HEAD(&acpi_desc->idts); 2765 INIT_LIST_HEAD(&acpi_desc->flushes); 2766 INIT_LIST_HEAD(&acpi_desc->memdevs); 2767 INIT_LIST_HEAD(&acpi_desc->dimms); 2768 INIT_LIST_HEAD(&acpi_desc->list); 2769 mutex_init(&acpi_desc->init_mutex); 2770 INIT_WORK(&acpi_desc->work, acpi_nfit_scrub); 2771 } 2772 EXPORT_SYMBOL_GPL(acpi_nfit_desc_init); 2773 2774 static int acpi_nfit_add(struct acpi_device *adev) 2775 { 2776 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 2777 struct acpi_nfit_desc *acpi_desc; 2778 struct device *dev = &adev->dev; 2779 struct acpi_table_header *tbl; 2780 acpi_status status = AE_OK; 2781 acpi_size sz; 2782 int rc = 0; 2783 2784 status = acpi_get_table_with_size(ACPI_SIG_NFIT, 0, &tbl, &sz); 2785 if (ACPI_FAILURE(status)) { 2786 /* This is ok, we could have an nvdimm hotplugged later */ 2787 dev_dbg(dev, "failed to find NFIT at startup\n"); 2788 return 0; 2789 } 2790 2791 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL); 2792 if (!acpi_desc) 2793 return -ENOMEM; 2794 acpi_nfit_desc_init(acpi_desc, &adev->dev); 2795 2796 /* Save the acpi header for exporting the revision via sysfs */ 2797 acpi_desc->acpi_header = *tbl; 2798 2799 /* Evaluate _FIT and override with that if present */ 2800 status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf); 2801 if (ACPI_SUCCESS(status) && buf.length > 0) { 2802 union acpi_object *obj = buf.pointer; 2803 2804 if (obj->type == ACPI_TYPE_BUFFER) 2805 rc = acpi_nfit_init(acpi_desc, obj->buffer.pointer, 2806 obj->buffer.length); 2807 else 2808 dev_dbg(dev, "%s invalid type %d, ignoring _FIT\n", 2809 __func__, (int) obj->type); 2810 kfree(buf.pointer); 2811 } else 2812 /* skip over the lead-in header table */ 2813 rc = acpi_nfit_init(acpi_desc, (void *) tbl 2814 + sizeof(struct acpi_table_nfit), 2815 sz - sizeof(struct acpi_table_nfit)); 2816 return rc; 2817 } 2818 2819 static int acpi_nfit_remove(struct acpi_device *adev) 2820 { 2821 /* see acpi_nfit_destruct */ 2822 return 0; 2823 } 2824 2825 void __acpi_nfit_notify(struct device *dev, acpi_handle handle, u32 event) 2826 { 2827 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev); 2828 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 2829 union acpi_object *obj; 2830 acpi_status status; 2831 int ret; 2832 2833 dev_dbg(dev, "%s: event: %d\n", __func__, event); 2834 2835 if (event != NFIT_NOTIFY_UPDATE) 2836 return; 2837 2838 if (!dev->driver) { 2839 /* dev->driver may be null if we're being removed */ 2840 dev_dbg(dev, "%s: no driver found for dev\n", __func__); 2841 return; 2842 } 2843 2844 if (!acpi_desc) { 2845 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL); 2846 if (!acpi_desc) 2847 return; 2848 acpi_nfit_desc_init(acpi_desc, dev); 2849 } else { 2850 /* 2851 * Finish previous registration before considering new 2852 * regions. 2853 */ 2854 flush_workqueue(nfit_wq); 2855 } 2856 2857 /* Evaluate _FIT */ 2858 status = acpi_evaluate_object(handle, "_FIT", NULL, &buf); 2859 if (ACPI_FAILURE(status)) { 2860 dev_err(dev, "failed to evaluate _FIT\n"); 2861 return; 2862 } 2863 2864 obj = buf.pointer; 2865 if (obj->type == ACPI_TYPE_BUFFER) { 2866 ret = acpi_nfit_init(acpi_desc, obj->buffer.pointer, 2867 obj->buffer.length); 2868 if (ret) 2869 dev_err(dev, "failed to merge updated NFIT\n"); 2870 } else 2871 dev_err(dev, "Invalid _FIT\n"); 2872 kfree(buf.pointer); 2873 } 2874 EXPORT_SYMBOL_GPL(__acpi_nfit_notify); 2875 2876 static void acpi_nfit_notify(struct acpi_device *adev, u32 event) 2877 { 2878 device_lock(&adev->dev); 2879 __acpi_nfit_notify(&adev->dev, adev->handle, event); 2880 device_unlock(&adev->dev); 2881 } 2882 2883 static const struct acpi_device_id acpi_nfit_ids[] = { 2884 { "ACPI0012", 0 }, 2885 { "", 0 }, 2886 }; 2887 MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids); 2888 2889 static struct acpi_driver acpi_nfit_driver = { 2890 .name = KBUILD_MODNAME, 2891 .ids = acpi_nfit_ids, 2892 .ops = { 2893 .add = acpi_nfit_add, 2894 .remove = acpi_nfit_remove, 2895 .notify = acpi_nfit_notify, 2896 }, 2897 }; 2898 2899 static __init int nfit_init(void) 2900 { 2901 BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40); 2902 BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56); 2903 BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48); 2904 BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20); 2905 BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9); 2906 BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80); 2907 BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40); 2908 2909 acpi_str_to_uuid(UUID_VOLATILE_MEMORY, nfit_uuid[NFIT_SPA_VOLATILE]); 2910 acpi_str_to_uuid(UUID_PERSISTENT_MEMORY, nfit_uuid[NFIT_SPA_PM]); 2911 acpi_str_to_uuid(UUID_CONTROL_REGION, nfit_uuid[NFIT_SPA_DCR]); 2912 acpi_str_to_uuid(UUID_DATA_REGION, nfit_uuid[NFIT_SPA_BDW]); 2913 acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_VDISK]); 2914 acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_CD, nfit_uuid[NFIT_SPA_VCD]); 2915 acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_PDISK]); 2916 acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_CD, nfit_uuid[NFIT_SPA_PCD]); 2917 acpi_str_to_uuid(UUID_NFIT_BUS, nfit_uuid[NFIT_DEV_BUS]); 2918 acpi_str_to_uuid(UUID_NFIT_DIMM, nfit_uuid[NFIT_DEV_DIMM]); 2919 acpi_str_to_uuid(UUID_NFIT_DIMM_N_HPE1, nfit_uuid[NFIT_DEV_DIMM_N_HPE1]); 2920 acpi_str_to_uuid(UUID_NFIT_DIMM_N_HPE2, nfit_uuid[NFIT_DEV_DIMM_N_HPE2]); 2921 acpi_str_to_uuid(UUID_NFIT_DIMM_N_MSFT, nfit_uuid[NFIT_DEV_DIMM_N_MSFT]); 2922 2923 nfit_wq = create_singlethread_workqueue("nfit"); 2924 if (!nfit_wq) 2925 return -ENOMEM; 2926 2927 nfit_mce_register(); 2928 2929 return acpi_bus_register_driver(&acpi_nfit_driver); 2930 } 2931 2932 static __exit void nfit_exit(void) 2933 { 2934 nfit_mce_unregister(); 2935 acpi_bus_unregister_driver(&acpi_nfit_driver); 2936 destroy_workqueue(nfit_wq); 2937 WARN_ON(!list_empty(&acpi_descs)); 2938 } 2939 2940 module_init(nfit_init); 2941 module_exit(nfit_exit); 2942 MODULE_LICENSE("GPL v2"); 2943 MODULE_AUTHOR("Intel Corporation"); 2944