1 /* 2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of version 2 of the GNU General Public License as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 */ 13 #include <linux/list_sort.h> 14 #include <linux/libnvdimm.h> 15 #include <linux/module.h> 16 #include <linux/mutex.h> 17 #include <linux/ndctl.h> 18 #include <linux/sysfs.h> 19 #include <linux/delay.h> 20 #include <linux/list.h> 21 #include <linux/acpi.h> 22 #include <linux/sort.h> 23 #include <linux/io.h> 24 #include <linux/nd.h> 25 #include <asm/cacheflush.h> 26 #include "nfit.h" 27 28 /* 29 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is 30 * irrelevant. 31 */ 32 #include <linux/io-64-nonatomic-hi-lo.h> 33 34 static bool force_enable_dimms; 35 module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR); 36 MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status"); 37 38 static unsigned int scrub_timeout = NFIT_ARS_TIMEOUT; 39 module_param(scrub_timeout, uint, S_IRUGO|S_IWUSR); 40 MODULE_PARM_DESC(scrub_timeout, "Initial scrub timeout in seconds"); 41 42 /* after three payloads of overflow, it's dead jim */ 43 static unsigned int scrub_overflow_abort = 3; 44 module_param(scrub_overflow_abort, uint, S_IRUGO|S_IWUSR); 45 MODULE_PARM_DESC(scrub_overflow_abort, 46 "Number of times we overflow ARS results before abort"); 47 48 static bool disable_vendor_specific; 49 module_param(disable_vendor_specific, bool, S_IRUGO); 50 MODULE_PARM_DESC(disable_vendor_specific, 51 "Limit commands to the publicly specified set"); 52 53 static unsigned long override_dsm_mask; 54 module_param(override_dsm_mask, ulong, S_IRUGO); 55 MODULE_PARM_DESC(override_dsm_mask, "Bitmask of allowed NVDIMM DSM functions"); 56 57 static int default_dsm_family = -1; 58 module_param(default_dsm_family, int, S_IRUGO); 59 MODULE_PARM_DESC(default_dsm_family, 60 "Try this DSM type first when identifying NVDIMM family"); 61 62 LIST_HEAD(acpi_descs); 63 DEFINE_MUTEX(acpi_desc_lock); 64 65 static struct workqueue_struct *nfit_wq; 66 67 struct nfit_table_prev { 68 struct list_head spas; 69 struct list_head memdevs; 70 struct list_head dcrs; 71 struct list_head bdws; 72 struct list_head idts; 73 struct list_head flushes; 74 }; 75 76 static guid_t nfit_uuid[NFIT_UUID_MAX]; 77 78 const guid_t *to_nfit_uuid(enum nfit_uuids id) 79 { 80 return &nfit_uuid[id]; 81 } 82 EXPORT_SYMBOL(to_nfit_uuid); 83 84 static struct acpi_nfit_desc *to_acpi_nfit_desc( 85 struct nvdimm_bus_descriptor *nd_desc) 86 { 87 return container_of(nd_desc, struct acpi_nfit_desc, nd_desc); 88 } 89 90 static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc) 91 { 92 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 93 94 /* 95 * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct 96 * acpi_device. 97 */ 98 if (!nd_desc->provider_name 99 || strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0) 100 return NULL; 101 102 return to_acpi_device(acpi_desc->dev); 103 } 104 105 static int xlat_bus_status(void *buf, unsigned int cmd, u32 status) 106 { 107 struct nd_cmd_clear_error *clear_err; 108 struct nd_cmd_ars_status *ars_status; 109 u16 flags; 110 111 switch (cmd) { 112 case ND_CMD_ARS_CAP: 113 if ((status & 0xffff) == NFIT_ARS_CAP_NONE) 114 return -ENOTTY; 115 116 /* Command failed */ 117 if (status & 0xffff) 118 return -EIO; 119 120 /* No supported scan types for this range */ 121 flags = ND_ARS_PERSISTENT | ND_ARS_VOLATILE; 122 if ((status >> 16 & flags) == 0) 123 return -ENOTTY; 124 return 0; 125 case ND_CMD_ARS_START: 126 /* ARS is in progress */ 127 if ((status & 0xffff) == NFIT_ARS_START_BUSY) 128 return -EBUSY; 129 130 /* Command failed */ 131 if (status & 0xffff) 132 return -EIO; 133 return 0; 134 case ND_CMD_ARS_STATUS: 135 ars_status = buf; 136 /* Command failed */ 137 if (status & 0xffff) 138 return -EIO; 139 /* Check extended status (Upper two bytes) */ 140 if (status == NFIT_ARS_STATUS_DONE) 141 return 0; 142 143 /* ARS is in progress */ 144 if (status == NFIT_ARS_STATUS_BUSY) 145 return -EBUSY; 146 147 /* No ARS performed for the current boot */ 148 if (status == NFIT_ARS_STATUS_NONE) 149 return -EAGAIN; 150 151 /* 152 * ARS interrupted, either we overflowed or some other 153 * agent wants the scan to stop. If we didn't overflow 154 * then just continue with the returned results. 155 */ 156 if (status == NFIT_ARS_STATUS_INTR) { 157 if (ars_status->out_length >= 40 && (ars_status->flags 158 & NFIT_ARS_F_OVERFLOW)) 159 return -ENOSPC; 160 return 0; 161 } 162 163 /* Unknown status */ 164 if (status >> 16) 165 return -EIO; 166 return 0; 167 case ND_CMD_CLEAR_ERROR: 168 clear_err = buf; 169 if (status & 0xffff) 170 return -EIO; 171 if (!clear_err->cleared) 172 return -EIO; 173 if (clear_err->length > clear_err->cleared) 174 return clear_err->cleared; 175 return 0; 176 default: 177 break; 178 } 179 180 /* all other non-zero status results in an error */ 181 if (status) 182 return -EIO; 183 return 0; 184 } 185 186 static int xlat_nvdimm_status(void *buf, unsigned int cmd, u32 status) 187 { 188 switch (cmd) { 189 case ND_CMD_GET_CONFIG_SIZE: 190 if (status >> 16 & ND_CONFIG_LOCKED) 191 return -EACCES; 192 break; 193 default: 194 break; 195 } 196 197 /* all other non-zero status results in an error */ 198 if (status) 199 return -EIO; 200 return 0; 201 } 202 203 static int xlat_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd, 204 u32 status) 205 { 206 if (!nvdimm) 207 return xlat_bus_status(buf, cmd, status); 208 return xlat_nvdimm_status(buf, cmd, status); 209 } 210 211 int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm, 212 unsigned int cmd, void *buf, unsigned int buf_len, int *cmd_rc) 213 { 214 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 215 union acpi_object in_obj, in_buf, *out_obj; 216 const struct nd_cmd_desc *desc = NULL; 217 struct device *dev = acpi_desc->dev; 218 struct nd_cmd_pkg *call_pkg = NULL; 219 const char *cmd_name, *dimm_name; 220 unsigned long cmd_mask, dsm_mask; 221 u32 offset, fw_status = 0; 222 acpi_handle handle; 223 unsigned int func; 224 const guid_t *guid; 225 int rc, i; 226 227 func = cmd; 228 if (cmd == ND_CMD_CALL) { 229 call_pkg = buf; 230 func = call_pkg->nd_command; 231 232 for (i = 0; i < ARRAY_SIZE(call_pkg->nd_reserved2); i++) 233 if (call_pkg->nd_reserved2[i]) 234 return -EINVAL; 235 } 236 237 if (nvdimm) { 238 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 239 struct acpi_device *adev = nfit_mem->adev; 240 241 if (!adev) 242 return -ENOTTY; 243 if (call_pkg && nfit_mem->family != call_pkg->nd_family) 244 return -ENOTTY; 245 246 dimm_name = nvdimm_name(nvdimm); 247 cmd_name = nvdimm_cmd_name(cmd); 248 cmd_mask = nvdimm_cmd_mask(nvdimm); 249 dsm_mask = nfit_mem->dsm_mask; 250 desc = nd_cmd_dimm_desc(cmd); 251 guid = to_nfit_uuid(nfit_mem->family); 252 handle = adev->handle; 253 } else { 254 struct acpi_device *adev = to_acpi_dev(acpi_desc); 255 256 cmd_name = nvdimm_bus_cmd_name(cmd); 257 cmd_mask = nd_desc->cmd_mask; 258 dsm_mask = cmd_mask; 259 if (cmd == ND_CMD_CALL) 260 dsm_mask = nd_desc->bus_dsm_mask; 261 desc = nd_cmd_bus_desc(cmd); 262 guid = to_nfit_uuid(NFIT_DEV_BUS); 263 handle = adev->handle; 264 dimm_name = "bus"; 265 } 266 267 if (!desc || (cmd && (desc->out_num + desc->in_num == 0))) 268 return -ENOTTY; 269 270 if (!test_bit(cmd, &cmd_mask) || !test_bit(func, &dsm_mask)) 271 return -ENOTTY; 272 273 in_obj.type = ACPI_TYPE_PACKAGE; 274 in_obj.package.count = 1; 275 in_obj.package.elements = &in_buf; 276 in_buf.type = ACPI_TYPE_BUFFER; 277 in_buf.buffer.pointer = buf; 278 in_buf.buffer.length = 0; 279 280 /* libnvdimm has already validated the input envelope */ 281 for (i = 0; i < desc->in_num; i++) 282 in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc, 283 i, buf); 284 285 if (call_pkg) { 286 /* skip over package wrapper */ 287 in_buf.buffer.pointer = (void *) &call_pkg->nd_payload; 288 in_buf.buffer.length = call_pkg->nd_size_in; 289 } 290 291 dev_dbg(dev, "%s:%s cmd: %d: func: %d input length: %d\n", 292 __func__, dimm_name, cmd, func, in_buf.buffer.length); 293 print_hex_dump_debug("nvdimm in ", DUMP_PREFIX_OFFSET, 4, 4, 294 in_buf.buffer.pointer, 295 min_t(u32, 256, in_buf.buffer.length), true); 296 297 out_obj = acpi_evaluate_dsm(handle, guid, 1, func, &in_obj); 298 if (!out_obj) { 299 dev_dbg(dev, "%s:%s _DSM failed cmd: %s\n", __func__, dimm_name, 300 cmd_name); 301 return -EINVAL; 302 } 303 304 if (call_pkg) { 305 call_pkg->nd_fw_size = out_obj->buffer.length; 306 memcpy(call_pkg->nd_payload + call_pkg->nd_size_in, 307 out_obj->buffer.pointer, 308 min(call_pkg->nd_fw_size, call_pkg->nd_size_out)); 309 310 ACPI_FREE(out_obj); 311 /* 312 * Need to support FW function w/o known size in advance. 313 * Caller can determine required size based upon nd_fw_size. 314 * If we return an error (like elsewhere) then caller wouldn't 315 * be able to rely upon data returned to make calculation. 316 */ 317 return 0; 318 } 319 320 if (out_obj->package.type != ACPI_TYPE_BUFFER) { 321 dev_dbg(dev, "%s:%s unexpected output object type cmd: %s type: %d\n", 322 __func__, dimm_name, cmd_name, out_obj->type); 323 rc = -EINVAL; 324 goto out; 325 } 326 327 dev_dbg(dev, "%s:%s cmd: %s output length: %d\n", __func__, dimm_name, 328 cmd_name, out_obj->buffer.length); 329 print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4, 4, 330 out_obj->buffer.pointer, 331 min_t(u32, 128, out_obj->buffer.length), true); 332 333 for (i = 0, offset = 0; i < desc->out_num; i++) { 334 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf, 335 (u32 *) out_obj->buffer.pointer, 336 out_obj->buffer.length - offset); 337 338 if (offset + out_size > out_obj->buffer.length) { 339 dev_dbg(dev, "%s:%s output object underflow cmd: %s field: %d\n", 340 __func__, dimm_name, cmd_name, i); 341 break; 342 } 343 344 if (in_buf.buffer.length + offset + out_size > buf_len) { 345 dev_dbg(dev, "%s:%s output overrun cmd: %s field: %d\n", 346 __func__, dimm_name, cmd_name, i); 347 rc = -ENXIO; 348 goto out; 349 } 350 memcpy(buf + in_buf.buffer.length + offset, 351 out_obj->buffer.pointer + offset, out_size); 352 offset += out_size; 353 } 354 355 /* 356 * Set fw_status for all the commands with a known format to be 357 * later interpreted by xlat_status(). 358 */ 359 if (i >= 1 && ((cmd >= ND_CMD_ARS_CAP && cmd <= ND_CMD_CLEAR_ERROR) 360 || (cmd >= ND_CMD_SMART && cmd <= ND_CMD_VENDOR))) 361 fw_status = *(u32 *) out_obj->buffer.pointer; 362 363 if (offset + in_buf.buffer.length < buf_len) { 364 if (i >= 1) { 365 /* 366 * status valid, return the number of bytes left 367 * unfilled in the output buffer 368 */ 369 rc = buf_len - offset - in_buf.buffer.length; 370 if (cmd_rc) 371 *cmd_rc = xlat_status(nvdimm, buf, cmd, 372 fw_status); 373 } else { 374 dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n", 375 __func__, dimm_name, cmd_name, buf_len, 376 offset); 377 rc = -ENXIO; 378 } 379 } else { 380 rc = 0; 381 if (cmd_rc) 382 *cmd_rc = xlat_status(nvdimm, buf, cmd, fw_status); 383 } 384 385 out: 386 ACPI_FREE(out_obj); 387 388 return rc; 389 } 390 EXPORT_SYMBOL_GPL(acpi_nfit_ctl); 391 392 static const char *spa_type_name(u16 type) 393 { 394 static const char *to_name[] = { 395 [NFIT_SPA_VOLATILE] = "volatile", 396 [NFIT_SPA_PM] = "pmem", 397 [NFIT_SPA_DCR] = "dimm-control-region", 398 [NFIT_SPA_BDW] = "block-data-window", 399 [NFIT_SPA_VDISK] = "volatile-disk", 400 [NFIT_SPA_VCD] = "volatile-cd", 401 [NFIT_SPA_PDISK] = "persistent-disk", 402 [NFIT_SPA_PCD] = "persistent-cd", 403 404 }; 405 406 if (type > NFIT_SPA_PCD) 407 return "unknown"; 408 409 return to_name[type]; 410 } 411 412 int nfit_spa_type(struct acpi_nfit_system_address *spa) 413 { 414 int i; 415 416 for (i = 0; i < NFIT_UUID_MAX; i++) 417 if (guid_equal(to_nfit_uuid(i), (guid_t *)&spa->range_guid)) 418 return i; 419 return -1; 420 } 421 422 static bool add_spa(struct acpi_nfit_desc *acpi_desc, 423 struct nfit_table_prev *prev, 424 struct acpi_nfit_system_address *spa) 425 { 426 struct device *dev = acpi_desc->dev; 427 struct nfit_spa *nfit_spa; 428 429 if (spa->header.length != sizeof(*spa)) 430 return false; 431 432 list_for_each_entry(nfit_spa, &prev->spas, list) { 433 if (memcmp(nfit_spa->spa, spa, sizeof(*spa)) == 0) { 434 list_move_tail(&nfit_spa->list, &acpi_desc->spas); 435 return true; 436 } 437 } 438 439 nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa) + sizeof(*spa), 440 GFP_KERNEL); 441 if (!nfit_spa) 442 return false; 443 INIT_LIST_HEAD(&nfit_spa->list); 444 memcpy(nfit_spa->spa, spa, sizeof(*spa)); 445 list_add_tail(&nfit_spa->list, &acpi_desc->spas); 446 dev_dbg(dev, "%s: spa index: %d type: %s\n", __func__, 447 spa->range_index, 448 spa_type_name(nfit_spa_type(spa))); 449 return true; 450 } 451 452 static bool add_memdev(struct acpi_nfit_desc *acpi_desc, 453 struct nfit_table_prev *prev, 454 struct acpi_nfit_memory_map *memdev) 455 { 456 struct device *dev = acpi_desc->dev; 457 struct nfit_memdev *nfit_memdev; 458 459 if (memdev->header.length != sizeof(*memdev)) 460 return false; 461 462 list_for_each_entry(nfit_memdev, &prev->memdevs, list) 463 if (memcmp(nfit_memdev->memdev, memdev, sizeof(*memdev)) == 0) { 464 list_move_tail(&nfit_memdev->list, &acpi_desc->memdevs); 465 return true; 466 } 467 468 nfit_memdev = devm_kzalloc(dev, sizeof(*nfit_memdev) + sizeof(*memdev), 469 GFP_KERNEL); 470 if (!nfit_memdev) 471 return false; 472 INIT_LIST_HEAD(&nfit_memdev->list); 473 memcpy(nfit_memdev->memdev, memdev, sizeof(*memdev)); 474 list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs); 475 dev_dbg(dev, "%s: memdev handle: %#x spa: %d dcr: %d flags: %#x\n", 476 __func__, memdev->device_handle, memdev->range_index, 477 memdev->region_index, memdev->flags); 478 return true; 479 } 480 481 /* 482 * An implementation may provide a truncated control region if no block windows 483 * are defined. 484 */ 485 static size_t sizeof_dcr(struct acpi_nfit_control_region *dcr) 486 { 487 if (dcr->header.length < offsetof(struct acpi_nfit_control_region, 488 window_size)) 489 return 0; 490 if (dcr->windows) 491 return sizeof(*dcr); 492 return offsetof(struct acpi_nfit_control_region, window_size); 493 } 494 495 static bool add_dcr(struct acpi_nfit_desc *acpi_desc, 496 struct nfit_table_prev *prev, 497 struct acpi_nfit_control_region *dcr) 498 { 499 struct device *dev = acpi_desc->dev; 500 struct nfit_dcr *nfit_dcr; 501 502 if (!sizeof_dcr(dcr)) 503 return false; 504 505 list_for_each_entry(nfit_dcr, &prev->dcrs, list) 506 if (memcmp(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)) == 0) { 507 list_move_tail(&nfit_dcr->list, &acpi_desc->dcrs); 508 return true; 509 } 510 511 nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr) + sizeof(*dcr), 512 GFP_KERNEL); 513 if (!nfit_dcr) 514 return false; 515 INIT_LIST_HEAD(&nfit_dcr->list); 516 memcpy(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)); 517 list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs); 518 dev_dbg(dev, "%s: dcr index: %d windows: %d\n", __func__, 519 dcr->region_index, dcr->windows); 520 return true; 521 } 522 523 static bool add_bdw(struct acpi_nfit_desc *acpi_desc, 524 struct nfit_table_prev *prev, 525 struct acpi_nfit_data_region *bdw) 526 { 527 struct device *dev = acpi_desc->dev; 528 struct nfit_bdw *nfit_bdw; 529 530 if (bdw->header.length != sizeof(*bdw)) 531 return false; 532 list_for_each_entry(nfit_bdw, &prev->bdws, list) 533 if (memcmp(nfit_bdw->bdw, bdw, sizeof(*bdw)) == 0) { 534 list_move_tail(&nfit_bdw->list, &acpi_desc->bdws); 535 return true; 536 } 537 538 nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw) + sizeof(*bdw), 539 GFP_KERNEL); 540 if (!nfit_bdw) 541 return false; 542 INIT_LIST_HEAD(&nfit_bdw->list); 543 memcpy(nfit_bdw->bdw, bdw, sizeof(*bdw)); 544 list_add_tail(&nfit_bdw->list, &acpi_desc->bdws); 545 dev_dbg(dev, "%s: bdw dcr: %d windows: %d\n", __func__, 546 bdw->region_index, bdw->windows); 547 return true; 548 } 549 550 static size_t sizeof_idt(struct acpi_nfit_interleave *idt) 551 { 552 if (idt->header.length < sizeof(*idt)) 553 return 0; 554 return sizeof(*idt) + sizeof(u32) * (idt->line_count - 1); 555 } 556 557 static bool add_idt(struct acpi_nfit_desc *acpi_desc, 558 struct nfit_table_prev *prev, 559 struct acpi_nfit_interleave *idt) 560 { 561 struct device *dev = acpi_desc->dev; 562 struct nfit_idt *nfit_idt; 563 564 if (!sizeof_idt(idt)) 565 return false; 566 567 list_for_each_entry(nfit_idt, &prev->idts, list) { 568 if (sizeof_idt(nfit_idt->idt) != sizeof_idt(idt)) 569 continue; 570 571 if (memcmp(nfit_idt->idt, idt, sizeof_idt(idt)) == 0) { 572 list_move_tail(&nfit_idt->list, &acpi_desc->idts); 573 return true; 574 } 575 } 576 577 nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt) + sizeof_idt(idt), 578 GFP_KERNEL); 579 if (!nfit_idt) 580 return false; 581 INIT_LIST_HEAD(&nfit_idt->list); 582 memcpy(nfit_idt->idt, idt, sizeof_idt(idt)); 583 list_add_tail(&nfit_idt->list, &acpi_desc->idts); 584 dev_dbg(dev, "%s: idt index: %d num_lines: %d\n", __func__, 585 idt->interleave_index, idt->line_count); 586 return true; 587 } 588 589 static size_t sizeof_flush(struct acpi_nfit_flush_address *flush) 590 { 591 if (flush->header.length < sizeof(*flush)) 592 return 0; 593 return sizeof(*flush) + sizeof(u64) * (flush->hint_count - 1); 594 } 595 596 static bool add_flush(struct acpi_nfit_desc *acpi_desc, 597 struct nfit_table_prev *prev, 598 struct acpi_nfit_flush_address *flush) 599 { 600 struct device *dev = acpi_desc->dev; 601 struct nfit_flush *nfit_flush; 602 603 if (!sizeof_flush(flush)) 604 return false; 605 606 list_for_each_entry(nfit_flush, &prev->flushes, list) { 607 if (sizeof_flush(nfit_flush->flush) != sizeof_flush(flush)) 608 continue; 609 610 if (memcmp(nfit_flush->flush, flush, 611 sizeof_flush(flush)) == 0) { 612 list_move_tail(&nfit_flush->list, &acpi_desc->flushes); 613 return true; 614 } 615 } 616 617 nfit_flush = devm_kzalloc(dev, sizeof(*nfit_flush) 618 + sizeof_flush(flush), GFP_KERNEL); 619 if (!nfit_flush) 620 return false; 621 INIT_LIST_HEAD(&nfit_flush->list); 622 memcpy(nfit_flush->flush, flush, sizeof_flush(flush)); 623 list_add_tail(&nfit_flush->list, &acpi_desc->flushes); 624 dev_dbg(dev, "%s: nfit_flush handle: %d hint_count: %d\n", __func__, 625 flush->device_handle, flush->hint_count); 626 return true; 627 } 628 629 static void *add_table(struct acpi_nfit_desc *acpi_desc, 630 struct nfit_table_prev *prev, void *table, const void *end) 631 { 632 struct device *dev = acpi_desc->dev; 633 struct acpi_nfit_header *hdr; 634 void *err = ERR_PTR(-ENOMEM); 635 636 if (table >= end) 637 return NULL; 638 639 hdr = table; 640 if (!hdr->length) { 641 dev_warn(dev, "found a zero length table '%d' parsing nfit\n", 642 hdr->type); 643 return NULL; 644 } 645 646 switch (hdr->type) { 647 case ACPI_NFIT_TYPE_SYSTEM_ADDRESS: 648 if (!add_spa(acpi_desc, prev, table)) 649 return err; 650 break; 651 case ACPI_NFIT_TYPE_MEMORY_MAP: 652 if (!add_memdev(acpi_desc, prev, table)) 653 return err; 654 break; 655 case ACPI_NFIT_TYPE_CONTROL_REGION: 656 if (!add_dcr(acpi_desc, prev, table)) 657 return err; 658 break; 659 case ACPI_NFIT_TYPE_DATA_REGION: 660 if (!add_bdw(acpi_desc, prev, table)) 661 return err; 662 break; 663 case ACPI_NFIT_TYPE_INTERLEAVE: 664 if (!add_idt(acpi_desc, prev, table)) 665 return err; 666 break; 667 case ACPI_NFIT_TYPE_FLUSH_ADDRESS: 668 if (!add_flush(acpi_desc, prev, table)) 669 return err; 670 break; 671 case ACPI_NFIT_TYPE_SMBIOS: 672 dev_dbg(dev, "%s: smbios\n", __func__); 673 break; 674 default: 675 dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type); 676 break; 677 } 678 679 return table + hdr->length; 680 } 681 682 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc, 683 struct nfit_mem *nfit_mem) 684 { 685 u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle; 686 u16 dcr = nfit_mem->dcr->region_index; 687 struct nfit_spa *nfit_spa; 688 689 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 690 u16 range_index = nfit_spa->spa->range_index; 691 int type = nfit_spa_type(nfit_spa->spa); 692 struct nfit_memdev *nfit_memdev; 693 694 if (type != NFIT_SPA_BDW) 695 continue; 696 697 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 698 if (nfit_memdev->memdev->range_index != range_index) 699 continue; 700 if (nfit_memdev->memdev->device_handle != device_handle) 701 continue; 702 if (nfit_memdev->memdev->region_index != dcr) 703 continue; 704 705 nfit_mem->spa_bdw = nfit_spa->spa; 706 return; 707 } 708 } 709 710 dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n", 711 nfit_mem->spa_dcr->range_index); 712 nfit_mem->bdw = NULL; 713 } 714 715 static void nfit_mem_init_bdw(struct acpi_nfit_desc *acpi_desc, 716 struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa) 717 { 718 u16 dcr = __to_nfit_memdev(nfit_mem)->region_index; 719 struct nfit_memdev *nfit_memdev; 720 struct nfit_bdw *nfit_bdw; 721 struct nfit_idt *nfit_idt; 722 u16 idt_idx, range_index; 723 724 list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) { 725 if (nfit_bdw->bdw->region_index != dcr) 726 continue; 727 nfit_mem->bdw = nfit_bdw->bdw; 728 break; 729 } 730 731 if (!nfit_mem->bdw) 732 return; 733 734 nfit_mem_find_spa_bdw(acpi_desc, nfit_mem); 735 736 if (!nfit_mem->spa_bdw) 737 return; 738 739 range_index = nfit_mem->spa_bdw->range_index; 740 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 741 if (nfit_memdev->memdev->range_index != range_index || 742 nfit_memdev->memdev->region_index != dcr) 743 continue; 744 nfit_mem->memdev_bdw = nfit_memdev->memdev; 745 idt_idx = nfit_memdev->memdev->interleave_index; 746 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) { 747 if (nfit_idt->idt->interleave_index != idt_idx) 748 continue; 749 nfit_mem->idt_bdw = nfit_idt->idt; 750 break; 751 } 752 break; 753 } 754 } 755 756 static int __nfit_mem_init(struct acpi_nfit_desc *acpi_desc, 757 struct acpi_nfit_system_address *spa) 758 { 759 struct nfit_mem *nfit_mem, *found; 760 struct nfit_memdev *nfit_memdev; 761 int type = spa ? nfit_spa_type(spa) : 0; 762 763 switch (type) { 764 case NFIT_SPA_DCR: 765 case NFIT_SPA_PM: 766 break; 767 default: 768 if (spa) 769 return 0; 770 } 771 772 /* 773 * This loop runs in two modes, when a dimm is mapped the loop 774 * adds memdev associations to an existing dimm, or creates a 775 * dimm. In the unmapped dimm case this loop sweeps for memdev 776 * instances with an invalid / zero range_index and adds those 777 * dimms without spa associations. 778 */ 779 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 780 struct nfit_flush *nfit_flush; 781 struct nfit_dcr *nfit_dcr; 782 u32 device_handle; 783 u16 dcr; 784 785 if (spa && nfit_memdev->memdev->range_index != spa->range_index) 786 continue; 787 if (!spa && nfit_memdev->memdev->range_index) 788 continue; 789 found = NULL; 790 dcr = nfit_memdev->memdev->region_index; 791 device_handle = nfit_memdev->memdev->device_handle; 792 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) 793 if (__to_nfit_memdev(nfit_mem)->device_handle 794 == device_handle) { 795 found = nfit_mem; 796 break; 797 } 798 799 if (found) 800 nfit_mem = found; 801 else { 802 nfit_mem = devm_kzalloc(acpi_desc->dev, 803 sizeof(*nfit_mem), GFP_KERNEL); 804 if (!nfit_mem) 805 return -ENOMEM; 806 INIT_LIST_HEAD(&nfit_mem->list); 807 nfit_mem->acpi_desc = acpi_desc; 808 list_add(&nfit_mem->list, &acpi_desc->dimms); 809 } 810 811 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) { 812 if (nfit_dcr->dcr->region_index != dcr) 813 continue; 814 /* 815 * Record the control region for the dimm. For 816 * the ACPI 6.1 case, where there are separate 817 * control regions for the pmem vs blk 818 * interfaces, be sure to record the extended 819 * blk details. 820 */ 821 if (!nfit_mem->dcr) 822 nfit_mem->dcr = nfit_dcr->dcr; 823 else if (nfit_mem->dcr->windows == 0 824 && nfit_dcr->dcr->windows) 825 nfit_mem->dcr = nfit_dcr->dcr; 826 break; 827 } 828 829 list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) { 830 struct acpi_nfit_flush_address *flush; 831 u16 i; 832 833 if (nfit_flush->flush->device_handle != device_handle) 834 continue; 835 nfit_mem->nfit_flush = nfit_flush; 836 flush = nfit_flush->flush; 837 nfit_mem->flush_wpq = devm_kzalloc(acpi_desc->dev, 838 flush->hint_count 839 * sizeof(struct resource), GFP_KERNEL); 840 if (!nfit_mem->flush_wpq) 841 return -ENOMEM; 842 for (i = 0; i < flush->hint_count; i++) { 843 struct resource *res = &nfit_mem->flush_wpq[i]; 844 845 res->start = flush->hint_address[i]; 846 res->end = res->start + 8 - 1; 847 } 848 break; 849 } 850 851 if (dcr && !nfit_mem->dcr) { 852 dev_err(acpi_desc->dev, "SPA %d missing DCR %d\n", 853 spa->range_index, dcr); 854 return -ENODEV; 855 } 856 857 if (type == NFIT_SPA_DCR) { 858 struct nfit_idt *nfit_idt; 859 u16 idt_idx; 860 861 /* multiple dimms may share a SPA when interleaved */ 862 nfit_mem->spa_dcr = spa; 863 nfit_mem->memdev_dcr = nfit_memdev->memdev; 864 idt_idx = nfit_memdev->memdev->interleave_index; 865 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) { 866 if (nfit_idt->idt->interleave_index != idt_idx) 867 continue; 868 nfit_mem->idt_dcr = nfit_idt->idt; 869 break; 870 } 871 nfit_mem_init_bdw(acpi_desc, nfit_mem, spa); 872 } else if (type == NFIT_SPA_PM) { 873 /* 874 * A single dimm may belong to multiple SPA-PM 875 * ranges, record at least one in addition to 876 * any SPA-DCR range. 877 */ 878 nfit_mem->memdev_pmem = nfit_memdev->memdev; 879 } else 880 nfit_mem->memdev_dcr = nfit_memdev->memdev; 881 } 882 883 return 0; 884 } 885 886 static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b) 887 { 888 struct nfit_mem *a = container_of(_a, typeof(*a), list); 889 struct nfit_mem *b = container_of(_b, typeof(*b), list); 890 u32 handleA, handleB; 891 892 handleA = __to_nfit_memdev(a)->device_handle; 893 handleB = __to_nfit_memdev(b)->device_handle; 894 if (handleA < handleB) 895 return -1; 896 else if (handleA > handleB) 897 return 1; 898 return 0; 899 } 900 901 static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc) 902 { 903 struct nfit_spa *nfit_spa; 904 int rc; 905 906 907 /* 908 * For each SPA-DCR or SPA-PMEM address range find its 909 * corresponding MEMDEV(s). From each MEMDEV find the 910 * corresponding DCR. Then, if we're operating on a SPA-DCR, 911 * try to find a SPA-BDW and a corresponding BDW that references 912 * the DCR. Throw it all into an nfit_mem object. Note, that 913 * BDWs are optional. 914 */ 915 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 916 rc = __nfit_mem_init(acpi_desc, nfit_spa->spa); 917 if (rc) 918 return rc; 919 } 920 921 /* 922 * If a DIMM has failed to be mapped into SPA there will be no 923 * SPA entries above. Find and register all the unmapped DIMMs 924 * for reporting and recovery purposes. 925 */ 926 rc = __nfit_mem_init(acpi_desc, NULL); 927 if (rc) 928 return rc; 929 930 list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp); 931 932 return 0; 933 } 934 935 static ssize_t bus_dsm_mask_show(struct device *dev, 936 struct device_attribute *attr, char *buf) 937 { 938 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 939 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 940 941 return sprintf(buf, "%#lx\n", nd_desc->bus_dsm_mask); 942 } 943 static struct device_attribute dev_attr_bus_dsm_mask = 944 __ATTR(dsm_mask, 0444, bus_dsm_mask_show, NULL); 945 946 static ssize_t revision_show(struct device *dev, 947 struct device_attribute *attr, char *buf) 948 { 949 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 950 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 951 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 952 953 return sprintf(buf, "%d\n", acpi_desc->acpi_header.revision); 954 } 955 static DEVICE_ATTR_RO(revision); 956 957 static ssize_t hw_error_scrub_show(struct device *dev, 958 struct device_attribute *attr, char *buf) 959 { 960 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 961 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 962 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 963 964 return sprintf(buf, "%d\n", acpi_desc->scrub_mode); 965 } 966 967 /* 968 * The 'hw_error_scrub' attribute can have the following values written to it: 969 * '0': Switch to the default mode where an exception will only insert 970 * the address of the memory error into the poison and badblocks lists. 971 * '1': Enable a full scrub to happen if an exception for a memory error is 972 * received. 973 */ 974 static ssize_t hw_error_scrub_store(struct device *dev, 975 struct device_attribute *attr, const char *buf, size_t size) 976 { 977 struct nvdimm_bus_descriptor *nd_desc; 978 ssize_t rc; 979 long val; 980 981 rc = kstrtol(buf, 0, &val); 982 if (rc) 983 return rc; 984 985 device_lock(dev); 986 nd_desc = dev_get_drvdata(dev); 987 if (nd_desc) { 988 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 989 990 switch (val) { 991 case HW_ERROR_SCRUB_ON: 992 acpi_desc->scrub_mode = HW_ERROR_SCRUB_ON; 993 break; 994 case HW_ERROR_SCRUB_OFF: 995 acpi_desc->scrub_mode = HW_ERROR_SCRUB_OFF; 996 break; 997 default: 998 rc = -EINVAL; 999 break; 1000 } 1001 } 1002 device_unlock(dev); 1003 if (rc) 1004 return rc; 1005 return size; 1006 } 1007 static DEVICE_ATTR_RW(hw_error_scrub); 1008 1009 /* 1010 * This shows the number of full Address Range Scrubs that have been 1011 * completed since driver load time. Userspace can wait on this using 1012 * select/poll etc. A '+' at the end indicates an ARS is in progress 1013 */ 1014 static ssize_t scrub_show(struct device *dev, 1015 struct device_attribute *attr, char *buf) 1016 { 1017 struct nvdimm_bus_descriptor *nd_desc; 1018 ssize_t rc = -ENXIO; 1019 1020 device_lock(dev); 1021 nd_desc = dev_get_drvdata(dev); 1022 if (nd_desc) { 1023 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1024 1025 rc = sprintf(buf, "%d%s", acpi_desc->scrub_count, 1026 (work_busy(&acpi_desc->work)) ? "+\n" : "\n"); 1027 } 1028 device_unlock(dev); 1029 return rc; 1030 } 1031 1032 static ssize_t scrub_store(struct device *dev, 1033 struct device_attribute *attr, const char *buf, size_t size) 1034 { 1035 struct nvdimm_bus_descriptor *nd_desc; 1036 ssize_t rc; 1037 long val; 1038 1039 rc = kstrtol(buf, 0, &val); 1040 if (rc) 1041 return rc; 1042 if (val != 1) 1043 return -EINVAL; 1044 1045 device_lock(dev); 1046 nd_desc = dev_get_drvdata(dev); 1047 if (nd_desc) { 1048 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); 1049 1050 rc = acpi_nfit_ars_rescan(acpi_desc, 0); 1051 } 1052 device_unlock(dev); 1053 if (rc) 1054 return rc; 1055 return size; 1056 } 1057 static DEVICE_ATTR_RW(scrub); 1058 1059 static bool ars_supported(struct nvdimm_bus *nvdimm_bus) 1060 { 1061 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 1062 const unsigned long mask = 1 << ND_CMD_ARS_CAP | 1 << ND_CMD_ARS_START 1063 | 1 << ND_CMD_ARS_STATUS; 1064 1065 return (nd_desc->cmd_mask & mask) == mask; 1066 } 1067 1068 static umode_t nfit_visible(struct kobject *kobj, struct attribute *a, int n) 1069 { 1070 struct device *dev = container_of(kobj, struct device, kobj); 1071 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); 1072 1073 if (a == &dev_attr_scrub.attr && !ars_supported(nvdimm_bus)) 1074 return 0; 1075 return a->mode; 1076 } 1077 1078 static struct attribute *acpi_nfit_attributes[] = { 1079 &dev_attr_revision.attr, 1080 &dev_attr_scrub.attr, 1081 &dev_attr_hw_error_scrub.attr, 1082 &dev_attr_bus_dsm_mask.attr, 1083 NULL, 1084 }; 1085 1086 static const struct attribute_group acpi_nfit_attribute_group = { 1087 .name = "nfit", 1088 .attrs = acpi_nfit_attributes, 1089 .is_visible = nfit_visible, 1090 }; 1091 1092 static const struct attribute_group *acpi_nfit_attribute_groups[] = { 1093 &nvdimm_bus_attribute_group, 1094 &acpi_nfit_attribute_group, 1095 NULL, 1096 }; 1097 1098 static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev) 1099 { 1100 struct nvdimm *nvdimm = to_nvdimm(dev); 1101 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1102 1103 return __to_nfit_memdev(nfit_mem); 1104 } 1105 1106 static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev) 1107 { 1108 struct nvdimm *nvdimm = to_nvdimm(dev); 1109 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1110 1111 return nfit_mem->dcr; 1112 } 1113 1114 static ssize_t handle_show(struct device *dev, 1115 struct device_attribute *attr, char *buf) 1116 { 1117 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev); 1118 1119 return sprintf(buf, "%#x\n", memdev->device_handle); 1120 } 1121 static DEVICE_ATTR_RO(handle); 1122 1123 static ssize_t phys_id_show(struct device *dev, 1124 struct device_attribute *attr, char *buf) 1125 { 1126 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev); 1127 1128 return sprintf(buf, "%#x\n", memdev->physical_id); 1129 } 1130 static DEVICE_ATTR_RO(phys_id); 1131 1132 static ssize_t vendor_show(struct device *dev, 1133 struct device_attribute *attr, char *buf) 1134 { 1135 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1136 1137 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->vendor_id)); 1138 } 1139 static DEVICE_ATTR_RO(vendor); 1140 1141 static ssize_t rev_id_show(struct device *dev, 1142 struct device_attribute *attr, char *buf) 1143 { 1144 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1145 1146 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->revision_id)); 1147 } 1148 static DEVICE_ATTR_RO(rev_id); 1149 1150 static ssize_t device_show(struct device *dev, 1151 struct device_attribute *attr, char *buf) 1152 { 1153 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1154 1155 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->device_id)); 1156 } 1157 static DEVICE_ATTR_RO(device); 1158 1159 static ssize_t subsystem_vendor_show(struct device *dev, 1160 struct device_attribute *attr, char *buf) 1161 { 1162 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1163 1164 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_vendor_id)); 1165 } 1166 static DEVICE_ATTR_RO(subsystem_vendor); 1167 1168 static ssize_t subsystem_rev_id_show(struct device *dev, 1169 struct device_attribute *attr, char *buf) 1170 { 1171 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1172 1173 return sprintf(buf, "0x%04x\n", 1174 be16_to_cpu(dcr->subsystem_revision_id)); 1175 } 1176 static DEVICE_ATTR_RO(subsystem_rev_id); 1177 1178 static ssize_t subsystem_device_show(struct device *dev, 1179 struct device_attribute *attr, char *buf) 1180 { 1181 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1182 1183 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_device_id)); 1184 } 1185 static DEVICE_ATTR_RO(subsystem_device); 1186 1187 static int num_nvdimm_formats(struct nvdimm *nvdimm) 1188 { 1189 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1190 int formats = 0; 1191 1192 if (nfit_mem->memdev_pmem) 1193 formats++; 1194 if (nfit_mem->memdev_bdw) 1195 formats++; 1196 return formats; 1197 } 1198 1199 static ssize_t format_show(struct device *dev, 1200 struct device_attribute *attr, char *buf) 1201 { 1202 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1203 1204 return sprintf(buf, "0x%04x\n", le16_to_cpu(dcr->code)); 1205 } 1206 static DEVICE_ATTR_RO(format); 1207 1208 static ssize_t format1_show(struct device *dev, 1209 struct device_attribute *attr, char *buf) 1210 { 1211 u32 handle; 1212 ssize_t rc = -ENXIO; 1213 struct nfit_mem *nfit_mem; 1214 struct nfit_memdev *nfit_memdev; 1215 struct acpi_nfit_desc *acpi_desc; 1216 struct nvdimm *nvdimm = to_nvdimm(dev); 1217 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1218 1219 nfit_mem = nvdimm_provider_data(nvdimm); 1220 acpi_desc = nfit_mem->acpi_desc; 1221 handle = to_nfit_memdev(dev)->device_handle; 1222 1223 /* assumes DIMMs have at most 2 published interface codes */ 1224 mutex_lock(&acpi_desc->init_mutex); 1225 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 1226 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev; 1227 struct nfit_dcr *nfit_dcr; 1228 1229 if (memdev->device_handle != handle) 1230 continue; 1231 1232 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) { 1233 if (nfit_dcr->dcr->region_index != memdev->region_index) 1234 continue; 1235 if (nfit_dcr->dcr->code == dcr->code) 1236 continue; 1237 rc = sprintf(buf, "0x%04x\n", 1238 le16_to_cpu(nfit_dcr->dcr->code)); 1239 break; 1240 } 1241 if (rc != ENXIO) 1242 break; 1243 } 1244 mutex_unlock(&acpi_desc->init_mutex); 1245 return rc; 1246 } 1247 static DEVICE_ATTR_RO(format1); 1248 1249 static ssize_t formats_show(struct device *dev, 1250 struct device_attribute *attr, char *buf) 1251 { 1252 struct nvdimm *nvdimm = to_nvdimm(dev); 1253 1254 return sprintf(buf, "%d\n", num_nvdimm_formats(nvdimm)); 1255 } 1256 static DEVICE_ATTR_RO(formats); 1257 1258 static ssize_t serial_show(struct device *dev, 1259 struct device_attribute *attr, char *buf) 1260 { 1261 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1262 1263 return sprintf(buf, "0x%08x\n", be32_to_cpu(dcr->serial_number)); 1264 } 1265 static DEVICE_ATTR_RO(serial); 1266 1267 static ssize_t family_show(struct device *dev, 1268 struct device_attribute *attr, char *buf) 1269 { 1270 struct nvdimm *nvdimm = to_nvdimm(dev); 1271 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1272 1273 if (nfit_mem->family < 0) 1274 return -ENXIO; 1275 return sprintf(buf, "%d\n", nfit_mem->family); 1276 } 1277 static DEVICE_ATTR_RO(family); 1278 1279 static ssize_t dsm_mask_show(struct device *dev, 1280 struct device_attribute *attr, char *buf) 1281 { 1282 struct nvdimm *nvdimm = to_nvdimm(dev); 1283 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1284 1285 if (nfit_mem->family < 0) 1286 return -ENXIO; 1287 return sprintf(buf, "%#lx\n", nfit_mem->dsm_mask); 1288 } 1289 static DEVICE_ATTR_RO(dsm_mask); 1290 1291 static ssize_t flags_show(struct device *dev, 1292 struct device_attribute *attr, char *buf) 1293 { 1294 u16 flags = to_nfit_memdev(dev)->flags; 1295 1296 return sprintf(buf, "%s%s%s%s%s%s%s\n", 1297 flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "", 1298 flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "", 1299 flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "", 1300 flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "", 1301 flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : "", 1302 flags & ACPI_NFIT_MEM_MAP_FAILED ? "map_fail " : "", 1303 flags & ACPI_NFIT_MEM_HEALTH_ENABLED ? "smart_notify " : ""); 1304 } 1305 static DEVICE_ATTR_RO(flags); 1306 1307 static ssize_t id_show(struct device *dev, 1308 struct device_attribute *attr, char *buf) 1309 { 1310 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); 1311 1312 if (dcr->valid_fields & ACPI_NFIT_CONTROL_MFG_INFO_VALID) 1313 return sprintf(buf, "%04x-%02x-%04x-%08x\n", 1314 be16_to_cpu(dcr->vendor_id), 1315 dcr->manufacturing_location, 1316 be16_to_cpu(dcr->manufacturing_date), 1317 be32_to_cpu(dcr->serial_number)); 1318 else 1319 return sprintf(buf, "%04x-%08x\n", 1320 be16_to_cpu(dcr->vendor_id), 1321 be32_to_cpu(dcr->serial_number)); 1322 } 1323 static DEVICE_ATTR_RO(id); 1324 1325 static struct attribute *acpi_nfit_dimm_attributes[] = { 1326 &dev_attr_handle.attr, 1327 &dev_attr_phys_id.attr, 1328 &dev_attr_vendor.attr, 1329 &dev_attr_device.attr, 1330 &dev_attr_rev_id.attr, 1331 &dev_attr_subsystem_vendor.attr, 1332 &dev_attr_subsystem_device.attr, 1333 &dev_attr_subsystem_rev_id.attr, 1334 &dev_attr_format.attr, 1335 &dev_attr_formats.attr, 1336 &dev_attr_format1.attr, 1337 &dev_attr_serial.attr, 1338 &dev_attr_flags.attr, 1339 &dev_attr_id.attr, 1340 &dev_attr_family.attr, 1341 &dev_attr_dsm_mask.attr, 1342 NULL, 1343 }; 1344 1345 static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj, 1346 struct attribute *a, int n) 1347 { 1348 struct device *dev = container_of(kobj, struct device, kobj); 1349 struct nvdimm *nvdimm = to_nvdimm(dev); 1350 1351 if (!to_nfit_dcr(dev)) { 1352 /* Without a dcr only the memdev attributes can be surfaced */ 1353 if (a == &dev_attr_handle.attr || a == &dev_attr_phys_id.attr 1354 || a == &dev_attr_flags.attr 1355 || a == &dev_attr_family.attr 1356 || a == &dev_attr_dsm_mask.attr) 1357 return a->mode; 1358 return 0; 1359 } 1360 1361 if (a == &dev_attr_format1.attr && num_nvdimm_formats(nvdimm) <= 1) 1362 return 0; 1363 return a->mode; 1364 } 1365 1366 static const struct attribute_group acpi_nfit_dimm_attribute_group = { 1367 .name = "nfit", 1368 .attrs = acpi_nfit_dimm_attributes, 1369 .is_visible = acpi_nfit_dimm_attr_visible, 1370 }; 1371 1372 static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = { 1373 &nvdimm_attribute_group, 1374 &nd_device_attribute_group, 1375 &acpi_nfit_dimm_attribute_group, 1376 NULL, 1377 }; 1378 1379 static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc, 1380 u32 device_handle) 1381 { 1382 struct nfit_mem *nfit_mem; 1383 1384 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) 1385 if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle) 1386 return nfit_mem->nvdimm; 1387 1388 return NULL; 1389 } 1390 1391 void __acpi_nvdimm_notify(struct device *dev, u32 event) 1392 { 1393 struct nfit_mem *nfit_mem; 1394 struct acpi_nfit_desc *acpi_desc; 1395 1396 dev_dbg(dev->parent, "%s: %s: event: %d\n", dev_name(dev), __func__, 1397 event); 1398 1399 if (event != NFIT_NOTIFY_DIMM_HEALTH) { 1400 dev_dbg(dev->parent, "%s: unknown event: %d\n", dev_name(dev), 1401 event); 1402 return; 1403 } 1404 1405 acpi_desc = dev_get_drvdata(dev->parent); 1406 if (!acpi_desc) 1407 return; 1408 1409 /* 1410 * If we successfully retrieved acpi_desc, then we know nfit_mem data 1411 * is still valid. 1412 */ 1413 nfit_mem = dev_get_drvdata(dev); 1414 if (nfit_mem && nfit_mem->flags_attr) 1415 sysfs_notify_dirent(nfit_mem->flags_attr); 1416 } 1417 EXPORT_SYMBOL_GPL(__acpi_nvdimm_notify); 1418 1419 static void acpi_nvdimm_notify(acpi_handle handle, u32 event, void *data) 1420 { 1421 struct acpi_device *adev = data; 1422 struct device *dev = &adev->dev; 1423 1424 device_lock(dev->parent); 1425 __acpi_nvdimm_notify(dev, event); 1426 device_unlock(dev->parent); 1427 } 1428 1429 static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc, 1430 struct nfit_mem *nfit_mem, u32 device_handle) 1431 { 1432 struct acpi_device *adev, *adev_dimm; 1433 struct device *dev = acpi_desc->dev; 1434 unsigned long dsm_mask; 1435 const guid_t *guid; 1436 int i; 1437 int family = -1; 1438 1439 /* nfit test assumes 1:1 relationship between commands and dsms */ 1440 nfit_mem->dsm_mask = acpi_desc->dimm_cmd_force_en; 1441 nfit_mem->family = NVDIMM_FAMILY_INTEL; 1442 adev = to_acpi_dev(acpi_desc); 1443 if (!adev) 1444 return 0; 1445 1446 adev_dimm = acpi_find_child_device(adev, device_handle, false); 1447 nfit_mem->adev = adev_dimm; 1448 if (!adev_dimm) { 1449 dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n", 1450 device_handle); 1451 return force_enable_dimms ? 0 : -ENODEV; 1452 } 1453 1454 if (ACPI_FAILURE(acpi_install_notify_handler(adev_dimm->handle, 1455 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify, adev_dimm))) { 1456 dev_err(dev, "%s: notification registration failed\n", 1457 dev_name(&adev_dimm->dev)); 1458 return -ENXIO; 1459 } 1460 1461 /* 1462 * Until standardization materializes we need to consider 4 1463 * different command sets. Note, that checking for function0 (bit0) 1464 * tells us if any commands are reachable through this GUID. 1465 */ 1466 for (i = NVDIMM_FAMILY_INTEL; i <= NVDIMM_FAMILY_MSFT; i++) 1467 if (acpi_check_dsm(adev_dimm->handle, to_nfit_uuid(i), 1, 1)) 1468 if (family < 0 || i == default_dsm_family) 1469 family = i; 1470 1471 /* limit the supported commands to those that are publicly documented */ 1472 nfit_mem->family = family; 1473 if (override_dsm_mask && !disable_vendor_specific) 1474 dsm_mask = override_dsm_mask; 1475 else if (nfit_mem->family == NVDIMM_FAMILY_INTEL) { 1476 dsm_mask = 0x3fe; 1477 if (disable_vendor_specific) 1478 dsm_mask &= ~(1 << ND_CMD_VENDOR); 1479 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE1) { 1480 dsm_mask = 0x1c3c76; 1481 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE2) { 1482 dsm_mask = 0x1fe; 1483 if (disable_vendor_specific) 1484 dsm_mask &= ~(1 << 8); 1485 } else if (nfit_mem->family == NVDIMM_FAMILY_MSFT) { 1486 dsm_mask = 0xffffffff; 1487 } else { 1488 dev_dbg(dev, "unknown dimm command family\n"); 1489 nfit_mem->family = -1; 1490 /* DSMs are optional, continue loading the driver... */ 1491 return 0; 1492 } 1493 1494 guid = to_nfit_uuid(nfit_mem->family); 1495 for_each_set_bit(i, &dsm_mask, BITS_PER_LONG) 1496 if (acpi_check_dsm(adev_dimm->handle, guid, 1, 1ULL << i)) 1497 set_bit(i, &nfit_mem->dsm_mask); 1498 1499 return 0; 1500 } 1501 1502 static void shutdown_dimm_notify(void *data) 1503 { 1504 struct acpi_nfit_desc *acpi_desc = data; 1505 struct nfit_mem *nfit_mem; 1506 1507 mutex_lock(&acpi_desc->init_mutex); 1508 /* 1509 * Clear out the nfit_mem->flags_attr and shut down dimm event 1510 * notifications. 1511 */ 1512 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1513 struct acpi_device *adev_dimm = nfit_mem->adev; 1514 1515 if (nfit_mem->flags_attr) { 1516 sysfs_put(nfit_mem->flags_attr); 1517 nfit_mem->flags_attr = NULL; 1518 } 1519 if (adev_dimm) 1520 acpi_remove_notify_handler(adev_dimm->handle, 1521 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify); 1522 } 1523 mutex_unlock(&acpi_desc->init_mutex); 1524 } 1525 1526 static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc) 1527 { 1528 struct nfit_mem *nfit_mem; 1529 int dimm_count = 0, rc; 1530 struct nvdimm *nvdimm; 1531 1532 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1533 struct acpi_nfit_flush_address *flush; 1534 unsigned long flags = 0, cmd_mask; 1535 struct nfit_memdev *nfit_memdev; 1536 u32 device_handle; 1537 u16 mem_flags; 1538 1539 device_handle = __to_nfit_memdev(nfit_mem)->device_handle; 1540 nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle); 1541 if (nvdimm) { 1542 dimm_count++; 1543 continue; 1544 } 1545 1546 if (nfit_mem->bdw && nfit_mem->memdev_pmem) 1547 set_bit(NDD_ALIASING, &flags); 1548 1549 /* collate flags across all memdevs for this dimm */ 1550 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 1551 struct acpi_nfit_memory_map *dimm_memdev; 1552 1553 dimm_memdev = __to_nfit_memdev(nfit_mem); 1554 if (dimm_memdev->device_handle 1555 != nfit_memdev->memdev->device_handle) 1556 continue; 1557 dimm_memdev->flags |= nfit_memdev->memdev->flags; 1558 } 1559 1560 mem_flags = __to_nfit_memdev(nfit_mem)->flags; 1561 if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED) 1562 set_bit(NDD_UNARMED, &flags); 1563 1564 rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle); 1565 if (rc) 1566 continue; 1567 1568 /* 1569 * TODO: provide translation for non-NVDIMM_FAMILY_INTEL 1570 * devices (i.e. from nd_cmd to acpi_dsm) to standardize the 1571 * userspace interface. 1572 */ 1573 cmd_mask = 1UL << ND_CMD_CALL; 1574 if (nfit_mem->family == NVDIMM_FAMILY_INTEL) 1575 cmd_mask |= nfit_mem->dsm_mask; 1576 1577 flush = nfit_mem->nfit_flush ? nfit_mem->nfit_flush->flush 1578 : NULL; 1579 nvdimm = nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem, 1580 acpi_nfit_dimm_attribute_groups, 1581 flags, cmd_mask, flush ? flush->hint_count : 0, 1582 nfit_mem->flush_wpq); 1583 if (!nvdimm) 1584 return -ENOMEM; 1585 1586 nfit_mem->nvdimm = nvdimm; 1587 dimm_count++; 1588 1589 if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0) 1590 continue; 1591 1592 dev_info(acpi_desc->dev, "%s flags:%s%s%s%s%s\n", 1593 nvdimm_name(nvdimm), 1594 mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "", 1595 mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"", 1596 mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "", 1597 mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : "", 1598 mem_flags & ACPI_NFIT_MEM_MAP_FAILED ? " map_fail" : ""); 1599 1600 } 1601 1602 rc = nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count); 1603 if (rc) 1604 return rc; 1605 1606 /* 1607 * Now that dimms are successfully registered, and async registration 1608 * is flushed, attempt to enable event notification. 1609 */ 1610 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { 1611 struct kernfs_node *nfit_kernfs; 1612 1613 nvdimm = nfit_mem->nvdimm; 1614 nfit_kernfs = sysfs_get_dirent(nvdimm_kobj(nvdimm)->sd, "nfit"); 1615 if (nfit_kernfs) 1616 nfit_mem->flags_attr = sysfs_get_dirent(nfit_kernfs, 1617 "flags"); 1618 sysfs_put(nfit_kernfs); 1619 if (!nfit_mem->flags_attr) 1620 dev_warn(acpi_desc->dev, "%s: notifications disabled\n", 1621 nvdimm_name(nvdimm)); 1622 } 1623 1624 return devm_add_action_or_reset(acpi_desc->dev, shutdown_dimm_notify, 1625 acpi_desc); 1626 } 1627 1628 /* 1629 * These constants are private because there are no kernel consumers of 1630 * these commands. 1631 */ 1632 enum nfit_aux_cmds { 1633 NFIT_CMD_TRANSLATE_SPA = 5, 1634 NFIT_CMD_ARS_INJECT_SET = 7, 1635 NFIT_CMD_ARS_INJECT_CLEAR = 8, 1636 NFIT_CMD_ARS_INJECT_GET = 9, 1637 }; 1638 1639 static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc) 1640 { 1641 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 1642 const guid_t *guid = to_nfit_uuid(NFIT_DEV_BUS); 1643 struct acpi_device *adev; 1644 unsigned long dsm_mask; 1645 int i; 1646 1647 nd_desc->cmd_mask = acpi_desc->bus_cmd_force_en; 1648 adev = to_acpi_dev(acpi_desc); 1649 if (!adev) 1650 return; 1651 1652 for (i = ND_CMD_ARS_CAP; i <= ND_CMD_CLEAR_ERROR; i++) 1653 if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i)) 1654 set_bit(i, &nd_desc->cmd_mask); 1655 set_bit(ND_CMD_CALL, &nd_desc->cmd_mask); 1656 1657 dsm_mask = 1658 (1 << ND_CMD_ARS_CAP) | 1659 (1 << ND_CMD_ARS_START) | 1660 (1 << ND_CMD_ARS_STATUS) | 1661 (1 << ND_CMD_CLEAR_ERROR) | 1662 (1 << NFIT_CMD_TRANSLATE_SPA) | 1663 (1 << NFIT_CMD_ARS_INJECT_SET) | 1664 (1 << NFIT_CMD_ARS_INJECT_CLEAR) | 1665 (1 << NFIT_CMD_ARS_INJECT_GET); 1666 for_each_set_bit(i, &dsm_mask, BITS_PER_LONG) 1667 if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i)) 1668 set_bit(i, &nd_desc->bus_dsm_mask); 1669 } 1670 1671 static ssize_t range_index_show(struct device *dev, 1672 struct device_attribute *attr, char *buf) 1673 { 1674 struct nd_region *nd_region = to_nd_region(dev); 1675 struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region); 1676 1677 return sprintf(buf, "%d\n", nfit_spa->spa->range_index); 1678 } 1679 static DEVICE_ATTR_RO(range_index); 1680 1681 static ssize_t ecc_unit_size_show(struct device *dev, 1682 struct device_attribute *attr, char *buf) 1683 { 1684 struct nd_region *nd_region = to_nd_region(dev); 1685 struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region); 1686 1687 return sprintf(buf, "%d\n", nfit_spa->clear_err_unit); 1688 } 1689 static DEVICE_ATTR_RO(ecc_unit_size); 1690 1691 static struct attribute *acpi_nfit_region_attributes[] = { 1692 &dev_attr_range_index.attr, 1693 &dev_attr_ecc_unit_size.attr, 1694 NULL, 1695 }; 1696 1697 static const struct attribute_group acpi_nfit_region_attribute_group = { 1698 .name = "nfit", 1699 .attrs = acpi_nfit_region_attributes, 1700 }; 1701 1702 static const struct attribute_group *acpi_nfit_region_attribute_groups[] = { 1703 &nd_region_attribute_group, 1704 &nd_mapping_attribute_group, 1705 &nd_device_attribute_group, 1706 &nd_numa_attribute_group, 1707 &acpi_nfit_region_attribute_group, 1708 NULL, 1709 }; 1710 1711 /* enough info to uniquely specify an interleave set */ 1712 struct nfit_set_info { 1713 struct nfit_set_info_map { 1714 u64 region_offset; 1715 u32 serial_number; 1716 u32 pad; 1717 } mapping[0]; 1718 }; 1719 1720 struct nfit_set_info2 { 1721 struct nfit_set_info_map2 { 1722 u64 region_offset; 1723 u32 serial_number; 1724 u16 vendor_id; 1725 u16 manufacturing_date; 1726 u8 manufacturing_location; 1727 u8 reserved[31]; 1728 } mapping[0]; 1729 }; 1730 1731 static size_t sizeof_nfit_set_info(int num_mappings) 1732 { 1733 return sizeof(struct nfit_set_info) 1734 + num_mappings * sizeof(struct nfit_set_info_map); 1735 } 1736 1737 static size_t sizeof_nfit_set_info2(int num_mappings) 1738 { 1739 return sizeof(struct nfit_set_info2) 1740 + num_mappings * sizeof(struct nfit_set_info_map2); 1741 } 1742 1743 static int cmp_map_compat(const void *m0, const void *m1) 1744 { 1745 const struct nfit_set_info_map *map0 = m0; 1746 const struct nfit_set_info_map *map1 = m1; 1747 1748 return memcmp(&map0->region_offset, &map1->region_offset, 1749 sizeof(u64)); 1750 } 1751 1752 static int cmp_map(const void *m0, const void *m1) 1753 { 1754 const struct nfit_set_info_map *map0 = m0; 1755 const struct nfit_set_info_map *map1 = m1; 1756 1757 if (map0->region_offset < map1->region_offset) 1758 return -1; 1759 else if (map0->region_offset > map1->region_offset) 1760 return 1; 1761 return 0; 1762 } 1763 1764 static int cmp_map2(const void *m0, const void *m1) 1765 { 1766 const struct nfit_set_info_map2 *map0 = m0; 1767 const struct nfit_set_info_map2 *map1 = m1; 1768 1769 if (map0->region_offset < map1->region_offset) 1770 return -1; 1771 else if (map0->region_offset > map1->region_offset) 1772 return 1; 1773 return 0; 1774 } 1775 1776 /* Retrieve the nth entry referencing this spa */ 1777 static struct acpi_nfit_memory_map *memdev_from_spa( 1778 struct acpi_nfit_desc *acpi_desc, u16 range_index, int n) 1779 { 1780 struct nfit_memdev *nfit_memdev; 1781 1782 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) 1783 if (nfit_memdev->memdev->range_index == range_index) 1784 if (n-- == 0) 1785 return nfit_memdev->memdev; 1786 return NULL; 1787 } 1788 1789 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc, 1790 struct nd_region_desc *ndr_desc, 1791 struct acpi_nfit_system_address *spa) 1792 { 1793 struct device *dev = acpi_desc->dev; 1794 struct nd_interleave_set *nd_set; 1795 u16 nr = ndr_desc->num_mappings; 1796 struct nfit_set_info2 *info2; 1797 struct nfit_set_info *info; 1798 int i; 1799 1800 nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL); 1801 if (!nd_set) 1802 return -ENOMEM; 1803 ndr_desc->nd_set = nd_set; 1804 guid_copy(&nd_set->type_guid, (guid_t *) spa->range_guid); 1805 1806 info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL); 1807 if (!info) 1808 return -ENOMEM; 1809 1810 info2 = devm_kzalloc(dev, sizeof_nfit_set_info2(nr), GFP_KERNEL); 1811 if (!info2) 1812 return -ENOMEM; 1813 1814 for (i = 0; i < nr; i++) { 1815 struct nd_mapping_desc *mapping = &ndr_desc->mapping[i]; 1816 struct nfit_set_info_map *map = &info->mapping[i]; 1817 struct nfit_set_info_map2 *map2 = &info2->mapping[i]; 1818 struct nvdimm *nvdimm = mapping->nvdimm; 1819 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1820 struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc, 1821 spa->range_index, i); 1822 struct acpi_nfit_control_region *dcr = nfit_mem->dcr; 1823 1824 if (!memdev || !nfit_mem->dcr) { 1825 dev_err(dev, "%s: failed to find DCR\n", __func__); 1826 return -ENODEV; 1827 } 1828 1829 map->region_offset = memdev->region_offset; 1830 map->serial_number = dcr->serial_number; 1831 1832 map2->region_offset = memdev->region_offset; 1833 map2->serial_number = dcr->serial_number; 1834 map2->vendor_id = dcr->vendor_id; 1835 map2->manufacturing_date = dcr->manufacturing_date; 1836 map2->manufacturing_location = dcr->manufacturing_location; 1837 } 1838 1839 /* v1.1 namespaces */ 1840 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map), 1841 cmp_map, NULL); 1842 nd_set->cookie1 = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0); 1843 1844 /* v1.2 namespaces */ 1845 sort(&info2->mapping[0], nr, sizeof(struct nfit_set_info_map2), 1846 cmp_map2, NULL); 1847 nd_set->cookie2 = nd_fletcher64(info2, sizeof_nfit_set_info2(nr), 0); 1848 1849 /* support v1.1 namespaces created with the wrong sort order */ 1850 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map), 1851 cmp_map_compat, NULL); 1852 nd_set->altcookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0); 1853 1854 /* record the result of the sort for the mapping position */ 1855 for (i = 0; i < nr; i++) { 1856 struct nfit_set_info_map2 *map2 = &info2->mapping[i]; 1857 int j; 1858 1859 for (j = 0; j < nr; j++) { 1860 struct nd_mapping_desc *mapping = &ndr_desc->mapping[j]; 1861 struct nvdimm *nvdimm = mapping->nvdimm; 1862 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); 1863 struct acpi_nfit_control_region *dcr = nfit_mem->dcr; 1864 1865 if (map2->serial_number == dcr->serial_number && 1866 map2->vendor_id == dcr->vendor_id && 1867 map2->manufacturing_date == dcr->manufacturing_date && 1868 map2->manufacturing_location 1869 == dcr->manufacturing_location) { 1870 mapping->position = i; 1871 break; 1872 } 1873 } 1874 } 1875 1876 ndr_desc->nd_set = nd_set; 1877 devm_kfree(dev, info); 1878 devm_kfree(dev, info2); 1879 1880 return 0; 1881 } 1882 1883 static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio) 1884 { 1885 struct acpi_nfit_interleave *idt = mmio->idt; 1886 u32 sub_line_offset, line_index, line_offset; 1887 u64 line_no, table_skip_count, table_offset; 1888 1889 line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset); 1890 table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index); 1891 line_offset = idt->line_offset[line_index] 1892 * mmio->line_size; 1893 table_offset = table_skip_count * mmio->table_size; 1894 1895 return mmio->base_offset + line_offset + table_offset + sub_line_offset; 1896 } 1897 1898 static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw) 1899 { 1900 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; 1901 u64 offset = nfit_blk->stat_offset + mmio->size * bw; 1902 const u32 STATUS_MASK = 0x80000037; 1903 1904 if (mmio->num_lines) 1905 offset = to_interleave_offset(offset, mmio); 1906 1907 return readl(mmio->addr.base + offset) & STATUS_MASK; 1908 } 1909 1910 static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw, 1911 resource_size_t dpa, unsigned int len, unsigned int write) 1912 { 1913 u64 cmd, offset; 1914 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; 1915 1916 enum { 1917 BCW_OFFSET_MASK = (1ULL << 48)-1, 1918 BCW_LEN_SHIFT = 48, 1919 BCW_LEN_MASK = (1ULL << 8) - 1, 1920 BCW_CMD_SHIFT = 56, 1921 }; 1922 1923 cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK; 1924 len = len >> L1_CACHE_SHIFT; 1925 cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT; 1926 cmd |= ((u64) write) << BCW_CMD_SHIFT; 1927 1928 offset = nfit_blk->cmd_offset + mmio->size * bw; 1929 if (mmio->num_lines) 1930 offset = to_interleave_offset(offset, mmio); 1931 1932 writeq(cmd, mmio->addr.base + offset); 1933 nvdimm_flush(nfit_blk->nd_region); 1934 1935 if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH) 1936 readq(mmio->addr.base + offset); 1937 } 1938 1939 static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk, 1940 resource_size_t dpa, void *iobuf, size_t len, int rw, 1941 unsigned int lane) 1942 { 1943 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW]; 1944 unsigned int copied = 0; 1945 u64 base_offset; 1946 int rc; 1947 1948 base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES 1949 + lane * mmio->size; 1950 write_blk_ctl(nfit_blk, lane, dpa, len, rw); 1951 while (len) { 1952 unsigned int c; 1953 u64 offset; 1954 1955 if (mmio->num_lines) { 1956 u32 line_offset; 1957 1958 offset = to_interleave_offset(base_offset + copied, 1959 mmio); 1960 div_u64_rem(offset, mmio->line_size, &line_offset); 1961 c = min_t(size_t, len, mmio->line_size - line_offset); 1962 } else { 1963 offset = base_offset + nfit_blk->bdw_offset; 1964 c = len; 1965 } 1966 1967 if (rw) 1968 memcpy_flushcache(mmio->addr.aperture + offset, iobuf + copied, c); 1969 else { 1970 if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH) 1971 arch_invalidate_pmem((void __force *) 1972 mmio->addr.aperture + offset, c); 1973 1974 memcpy(iobuf + copied, mmio->addr.aperture + offset, c); 1975 } 1976 1977 copied += c; 1978 len -= c; 1979 } 1980 1981 if (rw) 1982 nvdimm_flush(nfit_blk->nd_region); 1983 1984 rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0; 1985 return rc; 1986 } 1987 1988 static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr, 1989 resource_size_t dpa, void *iobuf, u64 len, int rw) 1990 { 1991 struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr); 1992 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW]; 1993 struct nd_region *nd_region = nfit_blk->nd_region; 1994 unsigned int lane, copied = 0; 1995 int rc = 0; 1996 1997 lane = nd_region_acquire_lane(nd_region); 1998 while (len) { 1999 u64 c = min(len, mmio->size); 2000 2001 rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied, 2002 iobuf + copied, c, rw, lane); 2003 if (rc) 2004 break; 2005 2006 copied += c; 2007 len -= c; 2008 } 2009 nd_region_release_lane(nd_region, lane); 2010 2011 return rc; 2012 } 2013 2014 static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio, 2015 struct acpi_nfit_interleave *idt, u16 interleave_ways) 2016 { 2017 if (idt) { 2018 mmio->num_lines = idt->line_count; 2019 mmio->line_size = idt->line_size; 2020 if (interleave_ways == 0) 2021 return -ENXIO; 2022 mmio->table_size = mmio->num_lines * interleave_ways 2023 * mmio->line_size; 2024 } 2025 2026 return 0; 2027 } 2028 2029 static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc, 2030 struct nvdimm *nvdimm, struct nfit_blk *nfit_blk) 2031 { 2032 struct nd_cmd_dimm_flags flags; 2033 int rc; 2034 2035 memset(&flags, 0, sizeof(flags)); 2036 rc = nd_desc->ndctl(nd_desc, nvdimm, ND_CMD_DIMM_FLAGS, &flags, 2037 sizeof(flags), NULL); 2038 2039 if (rc >= 0 && flags.status == 0) 2040 nfit_blk->dimm_flags = flags.flags; 2041 else if (rc == -ENOTTY) { 2042 /* fall back to a conservative default */ 2043 nfit_blk->dimm_flags = NFIT_BLK_DCR_LATCH | NFIT_BLK_READ_FLUSH; 2044 rc = 0; 2045 } else 2046 rc = -ENXIO; 2047 2048 return rc; 2049 } 2050 2051 static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, 2052 struct device *dev) 2053 { 2054 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); 2055 struct nd_blk_region *ndbr = to_nd_blk_region(dev); 2056 struct nfit_blk_mmio *mmio; 2057 struct nfit_blk *nfit_blk; 2058 struct nfit_mem *nfit_mem; 2059 struct nvdimm *nvdimm; 2060 int rc; 2061 2062 nvdimm = nd_blk_region_to_dimm(ndbr); 2063 nfit_mem = nvdimm_provider_data(nvdimm); 2064 if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) { 2065 dev_dbg(dev, "%s: missing%s%s%s\n", __func__, 2066 nfit_mem ? "" : " nfit_mem", 2067 (nfit_mem && nfit_mem->dcr) ? "" : " dcr", 2068 (nfit_mem && nfit_mem->bdw) ? "" : " bdw"); 2069 return -ENXIO; 2070 } 2071 2072 nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL); 2073 if (!nfit_blk) 2074 return -ENOMEM; 2075 nd_blk_region_set_provider_data(ndbr, nfit_blk); 2076 nfit_blk->nd_region = to_nd_region(dev); 2077 2078 /* map block aperture memory */ 2079 nfit_blk->bdw_offset = nfit_mem->bdw->offset; 2080 mmio = &nfit_blk->mmio[BDW]; 2081 mmio->addr.base = devm_nvdimm_memremap(dev, nfit_mem->spa_bdw->address, 2082 nfit_mem->spa_bdw->length, nd_blk_memremap_flags(ndbr)); 2083 if (!mmio->addr.base) { 2084 dev_dbg(dev, "%s: %s failed to map bdw\n", __func__, 2085 nvdimm_name(nvdimm)); 2086 return -ENOMEM; 2087 } 2088 mmio->size = nfit_mem->bdw->size; 2089 mmio->base_offset = nfit_mem->memdev_bdw->region_offset; 2090 mmio->idt = nfit_mem->idt_bdw; 2091 mmio->spa = nfit_mem->spa_bdw; 2092 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw, 2093 nfit_mem->memdev_bdw->interleave_ways); 2094 if (rc) { 2095 dev_dbg(dev, "%s: %s failed to init bdw interleave\n", 2096 __func__, nvdimm_name(nvdimm)); 2097 return rc; 2098 } 2099 2100 /* map block control memory */ 2101 nfit_blk->cmd_offset = nfit_mem->dcr->command_offset; 2102 nfit_blk->stat_offset = nfit_mem->dcr->status_offset; 2103 mmio = &nfit_blk->mmio[DCR]; 2104 mmio->addr.base = devm_nvdimm_ioremap(dev, nfit_mem->spa_dcr->address, 2105 nfit_mem->spa_dcr->length); 2106 if (!mmio->addr.base) { 2107 dev_dbg(dev, "%s: %s failed to map dcr\n", __func__, 2108 nvdimm_name(nvdimm)); 2109 return -ENOMEM; 2110 } 2111 mmio->size = nfit_mem->dcr->window_size; 2112 mmio->base_offset = nfit_mem->memdev_dcr->region_offset; 2113 mmio->idt = nfit_mem->idt_dcr; 2114 mmio->spa = nfit_mem->spa_dcr; 2115 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr, 2116 nfit_mem->memdev_dcr->interleave_ways); 2117 if (rc) { 2118 dev_dbg(dev, "%s: %s failed to init dcr interleave\n", 2119 __func__, nvdimm_name(nvdimm)); 2120 return rc; 2121 } 2122 2123 rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk); 2124 if (rc < 0) { 2125 dev_dbg(dev, "%s: %s failed get DIMM flags\n", 2126 __func__, nvdimm_name(nvdimm)); 2127 return rc; 2128 } 2129 2130 if (nvdimm_has_flush(nfit_blk->nd_region) < 0) 2131 dev_warn(dev, "unable to guarantee persistence of writes\n"); 2132 2133 if (mmio->line_size == 0) 2134 return 0; 2135 2136 if ((u32) nfit_blk->cmd_offset % mmio->line_size 2137 + 8 > mmio->line_size) { 2138 dev_dbg(dev, "cmd_offset crosses interleave boundary\n"); 2139 return -ENXIO; 2140 } else if ((u32) nfit_blk->stat_offset % mmio->line_size 2141 + 8 > mmio->line_size) { 2142 dev_dbg(dev, "stat_offset crosses interleave boundary\n"); 2143 return -ENXIO; 2144 } 2145 2146 return 0; 2147 } 2148 2149 static int ars_get_cap(struct acpi_nfit_desc *acpi_desc, 2150 struct nd_cmd_ars_cap *cmd, struct nfit_spa *nfit_spa) 2151 { 2152 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2153 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2154 int cmd_rc, rc; 2155 2156 cmd->address = spa->address; 2157 cmd->length = spa->length; 2158 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, cmd, 2159 sizeof(*cmd), &cmd_rc); 2160 if (rc < 0) 2161 return rc; 2162 return cmd_rc; 2163 } 2164 2165 static int ars_start(struct acpi_nfit_desc *acpi_desc, struct nfit_spa *nfit_spa) 2166 { 2167 int rc; 2168 int cmd_rc; 2169 struct nd_cmd_ars_start ars_start; 2170 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2171 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2172 2173 memset(&ars_start, 0, sizeof(ars_start)); 2174 ars_start.address = spa->address; 2175 ars_start.length = spa->length; 2176 ars_start.flags = acpi_desc->ars_start_flags; 2177 if (nfit_spa_type(spa) == NFIT_SPA_PM) 2178 ars_start.type = ND_ARS_PERSISTENT; 2179 else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE) 2180 ars_start.type = ND_ARS_VOLATILE; 2181 else 2182 return -ENOTTY; 2183 2184 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start, 2185 sizeof(ars_start), &cmd_rc); 2186 2187 if (rc < 0) 2188 return rc; 2189 return cmd_rc; 2190 } 2191 2192 static int ars_continue(struct acpi_nfit_desc *acpi_desc) 2193 { 2194 int rc, cmd_rc; 2195 struct nd_cmd_ars_start ars_start; 2196 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2197 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; 2198 2199 memset(&ars_start, 0, sizeof(ars_start)); 2200 ars_start.address = ars_status->restart_address; 2201 ars_start.length = ars_status->restart_length; 2202 ars_start.type = ars_status->type; 2203 ars_start.flags = acpi_desc->ars_start_flags; 2204 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start, 2205 sizeof(ars_start), &cmd_rc); 2206 if (rc < 0) 2207 return rc; 2208 return cmd_rc; 2209 } 2210 2211 static int ars_get_status(struct acpi_nfit_desc *acpi_desc) 2212 { 2213 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; 2214 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; 2215 int rc, cmd_rc; 2216 2217 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_STATUS, ars_status, 2218 acpi_desc->ars_status_size, &cmd_rc); 2219 if (rc < 0) 2220 return rc; 2221 return cmd_rc; 2222 } 2223 2224 static int ars_status_process_records(struct acpi_nfit_desc *acpi_desc, 2225 struct nd_cmd_ars_status *ars_status) 2226 { 2227 struct nvdimm_bus *nvdimm_bus = acpi_desc->nvdimm_bus; 2228 int rc; 2229 u32 i; 2230 2231 /* 2232 * First record starts at 44 byte offset from the start of the 2233 * payload. 2234 */ 2235 if (ars_status->out_length < 44) 2236 return 0; 2237 for (i = 0; i < ars_status->num_records; i++) { 2238 /* only process full records */ 2239 if (ars_status->out_length 2240 < 44 + sizeof(struct nd_ars_record) * (i + 1)) 2241 break; 2242 rc = nvdimm_bus_add_poison(nvdimm_bus, 2243 ars_status->records[i].err_address, 2244 ars_status->records[i].length); 2245 if (rc) 2246 return rc; 2247 } 2248 if (i < ars_status->num_records) 2249 dev_warn(acpi_desc->dev, "detected truncated ars results\n"); 2250 2251 return 0; 2252 } 2253 2254 static void acpi_nfit_remove_resource(void *data) 2255 { 2256 struct resource *res = data; 2257 2258 remove_resource(res); 2259 } 2260 2261 static int acpi_nfit_insert_resource(struct acpi_nfit_desc *acpi_desc, 2262 struct nd_region_desc *ndr_desc) 2263 { 2264 struct resource *res, *nd_res = ndr_desc->res; 2265 int is_pmem, ret; 2266 2267 /* No operation if the region is already registered as PMEM */ 2268 is_pmem = region_intersects(nd_res->start, resource_size(nd_res), 2269 IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY); 2270 if (is_pmem == REGION_INTERSECTS) 2271 return 0; 2272 2273 res = devm_kzalloc(acpi_desc->dev, sizeof(*res), GFP_KERNEL); 2274 if (!res) 2275 return -ENOMEM; 2276 2277 res->name = "Persistent Memory"; 2278 res->start = nd_res->start; 2279 res->end = nd_res->end; 2280 res->flags = IORESOURCE_MEM; 2281 res->desc = IORES_DESC_PERSISTENT_MEMORY; 2282 2283 ret = insert_resource(&iomem_resource, res); 2284 if (ret) 2285 return ret; 2286 2287 ret = devm_add_action_or_reset(acpi_desc->dev, 2288 acpi_nfit_remove_resource, 2289 res); 2290 if (ret) 2291 return ret; 2292 2293 return 0; 2294 } 2295 2296 static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc, 2297 struct nd_mapping_desc *mapping, struct nd_region_desc *ndr_desc, 2298 struct acpi_nfit_memory_map *memdev, 2299 struct nfit_spa *nfit_spa) 2300 { 2301 struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, 2302 memdev->device_handle); 2303 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2304 struct nd_blk_region_desc *ndbr_desc; 2305 struct nfit_mem *nfit_mem; 2306 int blk_valid = 0, rc; 2307 2308 if (!nvdimm) { 2309 dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n", 2310 spa->range_index, memdev->device_handle); 2311 return -ENODEV; 2312 } 2313 2314 mapping->nvdimm = nvdimm; 2315 switch (nfit_spa_type(spa)) { 2316 case NFIT_SPA_PM: 2317 case NFIT_SPA_VOLATILE: 2318 mapping->start = memdev->address; 2319 mapping->size = memdev->region_size; 2320 break; 2321 case NFIT_SPA_DCR: 2322 nfit_mem = nvdimm_provider_data(nvdimm); 2323 if (!nfit_mem || !nfit_mem->bdw) { 2324 dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n", 2325 spa->range_index, nvdimm_name(nvdimm)); 2326 } else { 2327 mapping->size = nfit_mem->bdw->capacity; 2328 mapping->start = nfit_mem->bdw->start_address; 2329 ndr_desc->num_lanes = nfit_mem->bdw->windows; 2330 blk_valid = 1; 2331 } 2332 2333 ndr_desc->mapping = mapping; 2334 ndr_desc->num_mappings = blk_valid; 2335 ndbr_desc = to_blk_region_desc(ndr_desc); 2336 ndbr_desc->enable = acpi_nfit_blk_region_enable; 2337 ndbr_desc->do_io = acpi_desc->blk_do_io; 2338 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa); 2339 if (rc) 2340 return rc; 2341 nfit_spa->nd_region = nvdimm_blk_region_create(acpi_desc->nvdimm_bus, 2342 ndr_desc); 2343 if (!nfit_spa->nd_region) 2344 return -ENOMEM; 2345 break; 2346 } 2347 2348 return 0; 2349 } 2350 2351 static bool nfit_spa_is_virtual(struct acpi_nfit_system_address *spa) 2352 { 2353 return (nfit_spa_type(spa) == NFIT_SPA_VDISK || 2354 nfit_spa_type(spa) == NFIT_SPA_VCD || 2355 nfit_spa_type(spa) == NFIT_SPA_PDISK || 2356 nfit_spa_type(spa) == NFIT_SPA_PCD); 2357 } 2358 2359 static bool nfit_spa_is_volatile(struct acpi_nfit_system_address *spa) 2360 { 2361 return (nfit_spa_type(spa) == NFIT_SPA_VDISK || 2362 nfit_spa_type(spa) == NFIT_SPA_VCD || 2363 nfit_spa_type(spa) == NFIT_SPA_VOLATILE); 2364 } 2365 2366 static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc, 2367 struct nfit_spa *nfit_spa) 2368 { 2369 static struct nd_mapping_desc mappings[ND_MAX_MAPPINGS]; 2370 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2371 struct nd_blk_region_desc ndbr_desc; 2372 struct nd_region_desc *ndr_desc; 2373 struct nfit_memdev *nfit_memdev; 2374 struct nvdimm_bus *nvdimm_bus; 2375 struct resource res; 2376 int count = 0, rc; 2377 2378 if (nfit_spa->nd_region) 2379 return 0; 2380 2381 if (spa->range_index == 0 && !nfit_spa_is_virtual(spa)) { 2382 dev_dbg(acpi_desc->dev, "%s: detected invalid spa index\n", 2383 __func__); 2384 return 0; 2385 } 2386 2387 memset(&res, 0, sizeof(res)); 2388 memset(&mappings, 0, sizeof(mappings)); 2389 memset(&ndbr_desc, 0, sizeof(ndbr_desc)); 2390 res.start = spa->address; 2391 res.end = res.start + spa->length - 1; 2392 ndr_desc = &ndbr_desc.ndr_desc; 2393 ndr_desc->res = &res; 2394 ndr_desc->provider_data = nfit_spa; 2395 ndr_desc->attr_groups = acpi_nfit_region_attribute_groups; 2396 if (spa->flags & ACPI_NFIT_PROXIMITY_VALID) 2397 ndr_desc->numa_node = acpi_map_pxm_to_online_node( 2398 spa->proximity_domain); 2399 else 2400 ndr_desc->numa_node = NUMA_NO_NODE; 2401 2402 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { 2403 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev; 2404 struct nd_mapping_desc *mapping; 2405 2406 if (memdev->range_index != spa->range_index) 2407 continue; 2408 if (count >= ND_MAX_MAPPINGS) { 2409 dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n", 2410 spa->range_index, ND_MAX_MAPPINGS); 2411 return -ENXIO; 2412 } 2413 mapping = &mappings[count++]; 2414 rc = acpi_nfit_init_mapping(acpi_desc, mapping, ndr_desc, 2415 memdev, nfit_spa); 2416 if (rc) 2417 goto out; 2418 } 2419 2420 ndr_desc->mapping = mappings; 2421 ndr_desc->num_mappings = count; 2422 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa); 2423 if (rc) 2424 goto out; 2425 2426 nvdimm_bus = acpi_desc->nvdimm_bus; 2427 if (nfit_spa_type(spa) == NFIT_SPA_PM) { 2428 rc = acpi_nfit_insert_resource(acpi_desc, ndr_desc); 2429 if (rc) { 2430 dev_warn(acpi_desc->dev, 2431 "failed to insert pmem resource to iomem: %d\n", 2432 rc); 2433 goto out; 2434 } 2435 2436 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus, 2437 ndr_desc); 2438 if (!nfit_spa->nd_region) 2439 rc = -ENOMEM; 2440 } else if (nfit_spa_is_volatile(spa)) { 2441 nfit_spa->nd_region = nvdimm_volatile_region_create(nvdimm_bus, 2442 ndr_desc); 2443 if (!nfit_spa->nd_region) 2444 rc = -ENOMEM; 2445 } else if (nfit_spa_is_virtual(spa)) { 2446 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus, 2447 ndr_desc); 2448 if (!nfit_spa->nd_region) 2449 rc = -ENOMEM; 2450 } 2451 2452 out: 2453 if (rc) 2454 dev_err(acpi_desc->dev, "failed to register spa range %d\n", 2455 nfit_spa->spa->range_index); 2456 return rc; 2457 } 2458 2459 static int ars_status_alloc(struct acpi_nfit_desc *acpi_desc, 2460 u32 max_ars) 2461 { 2462 struct device *dev = acpi_desc->dev; 2463 struct nd_cmd_ars_status *ars_status; 2464 2465 if (acpi_desc->ars_status && acpi_desc->ars_status_size >= max_ars) { 2466 memset(acpi_desc->ars_status, 0, acpi_desc->ars_status_size); 2467 return 0; 2468 } 2469 2470 if (acpi_desc->ars_status) 2471 devm_kfree(dev, acpi_desc->ars_status); 2472 acpi_desc->ars_status = NULL; 2473 ars_status = devm_kzalloc(dev, max_ars, GFP_KERNEL); 2474 if (!ars_status) 2475 return -ENOMEM; 2476 acpi_desc->ars_status = ars_status; 2477 acpi_desc->ars_status_size = max_ars; 2478 return 0; 2479 } 2480 2481 static int acpi_nfit_query_poison(struct acpi_nfit_desc *acpi_desc, 2482 struct nfit_spa *nfit_spa) 2483 { 2484 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2485 int rc; 2486 2487 if (!nfit_spa->max_ars) { 2488 struct nd_cmd_ars_cap ars_cap; 2489 2490 memset(&ars_cap, 0, sizeof(ars_cap)); 2491 rc = ars_get_cap(acpi_desc, &ars_cap, nfit_spa); 2492 if (rc < 0) 2493 return rc; 2494 nfit_spa->max_ars = ars_cap.max_ars_out; 2495 nfit_spa->clear_err_unit = ars_cap.clear_err_unit; 2496 /* check that the supported scrub types match the spa type */ 2497 if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE && 2498 ((ars_cap.status >> 16) & ND_ARS_VOLATILE) == 0) 2499 return -ENOTTY; 2500 else if (nfit_spa_type(spa) == NFIT_SPA_PM && 2501 ((ars_cap.status >> 16) & ND_ARS_PERSISTENT) == 0) 2502 return -ENOTTY; 2503 } 2504 2505 if (ars_status_alloc(acpi_desc, nfit_spa->max_ars)) 2506 return -ENOMEM; 2507 2508 rc = ars_get_status(acpi_desc); 2509 if (rc < 0 && rc != -ENOSPC) 2510 return rc; 2511 2512 if (ars_status_process_records(acpi_desc, acpi_desc->ars_status)) 2513 return -ENOMEM; 2514 2515 return 0; 2516 } 2517 2518 static void acpi_nfit_async_scrub(struct acpi_nfit_desc *acpi_desc, 2519 struct nfit_spa *nfit_spa) 2520 { 2521 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2522 unsigned int overflow_retry = scrub_overflow_abort; 2523 u64 init_ars_start = 0, init_ars_len = 0; 2524 struct device *dev = acpi_desc->dev; 2525 unsigned int tmo = scrub_timeout; 2526 int rc; 2527 2528 if (!nfit_spa->ars_required || !nfit_spa->nd_region) 2529 return; 2530 2531 rc = ars_start(acpi_desc, nfit_spa); 2532 /* 2533 * If we timed out the initial scan we'll still be busy here, 2534 * and will wait another timeout before giving up permanently. 2535 */ 2536 if (rc < 0 && rc != -EBUSY) 2537 return; 2538 2539 do { 2540 u64 ars_start, ars_len; 2541 2542 if (acpi_desc->cancel) 2543 break; 2544 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa); 2545 if (rc == -ENOTTY) 2546 break; 2547 if (rc == -EBUSY && !tmo) { 2548 dev_warn(dev, "range %d ars timeout, aborting\n", 2549 spa->range_index); 2550 break; 2551 } 2552 2553 if (rc == -EBUSY) { 2554 /* 2555 * Note, entries may be appended to the list 2556 * while the lock is dropped, but the workqueue 2557 * being active prevents entries being deleted / 2558 * freed. 2559 */ 2560 mutex_unlock(&acpi_desc->init_mutex); 2561 ssleep(1); 2562 tmo--; 2563 mutex_lock(&acpi_desc->init_mutex); 2564 continue; 2565 } 2566 2567 /* we got some results, but there are more pending... */ 2568 if (rc == -ENOSPC && overflow_retry--) { 2569 if (!init_ars_len) { 2570 init_ars_len = acpi_desc->ars_status->length; 2571 init_ars_start = acpi_desc->ars_status->address; 2572 } 2573 rc = ars_continue(acpi_desc); 2574 } 2575 2576 if (rc < 0) { 2577 dev_warn(dev, "range %d ars continuation failed\n", 2578 spa->range_index); 2579 break; 2580 } 2581 2582 if (init_ars_len) { 2583 ars_start = init_ars_start; 2584 ars_len = init_ars_len; 2585 } else { 2586 ars_start = acpi_desc->ars_status->address; 2587 ars_len = acpi_desc->ars_status->length; 2588 } 2589 dev_dbg(dev, "spa range: %d ars from %#llx + %#llx complete\n", 2590 spa->range_index, ars_start, ars_len); 2591 /* notify the region about new poison entries */ 2592 nvdimm_region_notify(nfit_spa->nd_region, 2593 NVDIMM_REVALIDATE_POISON); 2594 break; 2595 } while (1); 2596 } 2597 2598 static void acpi_nfit_scrub(struct work_struct *work) 2599 { 2600 struct device *dev; 2601 u64 init_scrub_length = 0; 2602 struct nfit_spa *nfit_spa; 2603 u64 init_scrub_address = 0; 2604 bool init_ars_done = false; 2605 struct acpi_nfit_desc *acpi_desc; 2606 unsigned int tmo = scrub_timeout; 2607 unsigned int overflow_retry = scrub_overflow_abort; 2608 2609 acpi_desc = container_of(work, typeof(*acpi_desc), work); 2610 dev = acpi_desc->dev; 2611 2612 /* 2613 * We scrub in 2 phases. The first phase waits for any platform 2614 * firmware initiated scrubs to complete and then we go search for the 2615 * affected spa regions to mark them scanned. In the second phase we 2616 * initiate a directed scrub for every range that was not scrubbed in 2617 * phase 1. If we're called for a 'rescan', we harmlessly pass through 2618 * the first phase, but really only care about running phase 2, where 2619 * regions can be notified of new poison. 2620 */ 2621 2622 /* process platform firmware initiated scrubs */ 2623 retry: 2624 mutex_lock(&acpi_desc->init_mutex); 2625 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 2626 struct nd_cmd_ars_status *ars_status; 2627 struct acpi_nfit_system_address *spa; 2628 u64 ars_start, ars_len; 2629 int rc; 2630 2631 if (acpi_desc->cancel) 2632 break; 2633 2634 if (nfit_spa->nd_region) 2635 continue; 2636 2637 if (init_ars_done) { 2638 /* 2639 * No need to re-query, we're now just 2640 * reconciling all the ranges covered by the 2641 * initial scrub 2642 */ 2643 rc = 0; 2644 } else 2645 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa); 2646 2647 if (rc == -ENOTTY) { 2648 /* no ars capability, just register spa and move on */ 2649 acpi_nfit_register_region(acpi_desc, nfit_spa); 2650 continue; 2651 } 2652 2653 if (rc == -EBUSY && !tmo) { 2654 /* fallthrough to directed scrub in phase 2 */ 2655 dev_warn(dev, "timeout awaiting ars results, continuing...\n"); 2656 break; 2657 } else if (rc == -EBUSY) { 2658 mutex_unlock(&acpi_desc->init_mutex); 2659 ssleep(1); 2660 tmo--; 2661 goto retry; 2662 } 2663 2664 /* we got some results, but there are more pending... */ 2665 if (rc == -ENOSPC && overflow_retry--) { 2666 ars_status = acpi_desc->ars_status; 2667 /* 2668 * Record the original scrub range, so that we 2669 * can recall all the ranges impacted by the 2670 * initial scrub. 2671 */ 2672 if (!init_scrub_length) { 2673 init_scrub_length = ars_status->length; 2674 init_scrub_address = ars_status->address; 2675 } 2676 rc = ars_continue(acpi_desc); 2677 if (rc == 0) { 2678 mutex_unlock(&acpi_desc->init_mutex); 2679 goto retry; 2680 } 2681 } 2682 2683 if (rc < 0) { 2684 /* 2685 * Initial scrub failed, we'll give it one more 2686 * try below... 2687 */ 2688 break; 2689 } 2690 2691 /* We got some final results, record completed ranges */ 2692 ars_status = acpi_desc->ars_status; 2693 if (init_scrub_length) { 2694 ars_start = init_scrub_address; 2695 ars_len = ars_start + init_scrub_length; 2696 } else { 2697 ars_start = ars_status->address; 2698 ars_len = ars_status->length; 2699 } 2700 spa = nfit_spa->spa; 2701 2702 if (!init_ars_done) { 2703 init_ars_done = true; 2704 dev_dbg(dev, "init scrub %#llx + %#llx complete\n", 2705 ars_start, ars_len); 2706 } 2707 if (ars_start <= spa->address && ars_start + ars_len 2708 >= spa->address + spa->length) 2709 acpi_nfit_register_region(acpi_desc, nfit_spa); 2710 } 2711 2712 /* 2713 * For all the ranges not covered by an initial scrub we still 2714 * want to see if there are errors, but it's ok to discover them 2715 * asynchronously. 2716 */ 2717 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 2718 /* 2719 * Flag all the ranges that still need scrubbing, but 2720 * register them now to make data available. 2721 */ 2722 if (!nfit_spa->nd_region) { 2723 nfit_spa->ars_required = 1; 2724 acpi_nfit_register_region(acpi_desc, nfit_spa); 2725 } 2726 } 2727 acpi_desc->init_complete = 1; 2728 2729 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) 2730 acpi_nfit_async_scrub(acpi_desc, nfit_spa); 2731 acpi_desc->scrub_count++; 2732 acpi_desc->ars_start_flags = 0; 2733 if (acpi_desc->scrub_count_state) 2734 sysfs_notify_dirent(acpi_desc->scrub_count_state); 2735 mutex_unlock(&acpi_desc->init_mutex); 2736 } 2737 2738 static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc) 2739 { 2740 struct nfit_spa *nfit_spa; 2741 int rc; 2742 2743 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) 2744 if (nfit_spa_type(nfit_spa->spa) == NFIT_SPA_DCR) { 2745 /* BLK regions don't need to wait for ars results */ 2746 rc = acpi_nfit_register_region(acpi_desc, nfit_spa); 2747 if (rc) 2748 return rc; 2749 } 2750 2751 acpi_desc->ars_start_flags = 0; 2752 if (!acpi_desc->cancel) 2753 queue_work(nfit_wq, &acpi_desc->work); 2754 return 0; 2755 } 2756 2757 static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc, 2758 struct nfit_table_prev *prev) 2759 { 2760 struct device *dev = acpi_desc->dev; 2761 2762 if (!list_empty(&prev->spas) || 2763 !list_empty(&prev->memdevs) || 2764 !list_empty(&prev->dcrs) || 2765 !list_empty(&prev->bdws) || 2766 !list_empty(&prev->idts) || 2767 !list_empty(&prev->flushes)) { 2768 dev_err(dev, "new nfit deletes entries (unsupported)\n"); 2769 return -ENXIO; 2770 } 2771 return 0; 2772 } 2773 2774 static int acpi_nfit_desc_init_scrub_attr(struct acpi_nfit_desc *acpi_desc) 2775 { 2776 struct device *dev = acpi_desc->dev; 2777 struct kernfs_node *nfit; 2778 struct device *bus_dev; 2779 2780 if (!ars_supported(acpi_desc->nvdimm_bus)) 2781 return 0; 2782 2783 bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); 2784 nfit = sysfs_get_dirent(bus_dev->kobj.sd, "nfit"); 2785 if (!nfit) { 2786 dev_err(dev, "sysfs_get_dirent 'nfit' failed\n"); 2787 return -ENODEV; 2788 } 2789 acpi_desc->scrub_count_state = sysfs_get_dirent(nfit, "scrub"); 2790 sysfs_put(nfit); 2791 if (!acpi_desc->scrub_count_state) { 2792 dev_err(dev, "sysfs_get_dirent 'scrub' failed\n"); 2793 return -ENODEV; 2794 } 2795 2796 return 0; 2797 } 2798 2799 static void acpi_nfit_unregister(void *data) 2800 { 2801 struct acpi_nfit_desc *acpi_desc = data; 2802 2803 nvdimm_bus_unregister(acpi_desc->nvdimm_bus); 2804 } 2805 2806 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz) 2807 { 2808 struct device *dev = acpi_desc->dev; 2809 struct nfit_table_prev prev; 2810 const void *end; 2811 int rc; 2812 2813 if (!acpi_desc->nvdimm_bus) { 2814 acpi_nfit_init_dsms(acpi_desc); 2815 2816 acpi_desc->nvdimm_bus = nvdimm_bus_register(dev, 2817 &acpi_desc->nd_desc); 2818 if (!acpi_desc->nvdimm_bus) 2819 return -ENOMEM; 2820 2821 rc = devm_add_action_or_reset(dev, acpi_nfit_unregister, 2822 acpi_desc); 2823 if (rc) 2824 return rc; 2825 2826 rc = acpi_nfit_desc_init_scrub_attr(acpi_desc); 2827 if (rc) 2828 return rc; 2829 2830 /* register this acpi_desc for mce notifications */ 2831 mutex_lock(&acpi_desc_lock); 2832 list_add_tail(&acpi_desc->list, &acpi_descs); 2833 mutex_unlock(&acpi_desc_lock); 2834 } 2835 2836 mutex_lock(&acpi_desc->init_mutex); 2837 2838 INIT_LIST_HEAD(&prev.spas); 2839 INIT_LIST_HEAD(&prev.memdevs); 2840 INIT_LIST_HEAD(&prev.dcrs); 2841 INIT_LIST_HEAD(&prev.bdws); 2842 INIT_LIST_HEAD(&prev.idts); 2843 INIT_LIST_HEAD(&prev.flushes); 2844 2845 list_cut_position(&prev.spas, &acpi_desc->spas, 2846 acpi_desc->spas.prev); 2847 list_cut_position(&prev.memdevs, &acpi_desc->memdevs, 2848 acpi_desc->memdevs.prev); 2849 list_cut_position(&prev.dcrs, &acpi_desc->dcrs, 2850 acpi_desc->dcrs.prev); 2851 list_cut_position(&prev.bdws, &acpi_desc->bdws, 2852 acpi_desc->bdws.prev); 2853 list_cut_position(&prev.idts, &acpi_desc->idts, 2854 acpi_desc->idts.prev); 2855 list_cut_position(&prev.flushes, &acpi_desc->flushes, 2856 acpi_desc->flushes.prev); 2857 2858 end = data + sz; 2859 while (!IS_ERR_OR_NULL(data)) 2860 data = add_table(acpi_desc, &prev, data, end); 2861 2862 if (IS_ERR(data)) { 2863 dev_dbg(dev, "%s: nfit table parsing error: %ld\n", __func__, 2864 PTR_ERR(data)); 2865 rc = PTR_ERR(data); 2866 goto out_unlock; 2867 } 2868 2869 rc = acpi_nfit_check_deletions(acpi_desc, &prev); 2870 if (rc) 2871 goto out_unlock; 2872 2873 rc = nfit_mem_init(acpi_desc); 2874 if (rc) 2875 goto out_unlock; 2876 2877 rc = acpi_nfit_register_dimms(acpi_desc); 2878 if (rc) 2879 goto out_unlock; 2880 2881 rc = acpi_nfit_register_regions(acpi_desc); 2882 2883 out_unlock: 2884 mutex_unlock(&acpi_desc->init_mutex); 2885 return rc; 2886 } 2887 EXPORT_SYMBOL_GPL(acpi_nfit_init); 2888 2889 struct acpi_nfit_flush_work { 2890 struct work_struct work; 2891 struct completion cmp; 2892 }; 2893 2894 static void flush_probe(struct work_struct *work) 2895 { 2896 struct acpi_nfit_flush_work *flush; 2897 2898 flush = container_of(work, typeof(*flush), work); 2899 complete(&flush->cmp); 2900 } 2901 2902 static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc) 2903 { 2904 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 2905 struct device *dev = acpi_desc->dev; 2906 struct acpi_nfit_flush_work flush; 2907 int rc; 2908 2909 /* bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */ 2910 device_lock(dev); 2911 device_unlock(dev); 2912 2913 /* bounce the init_mutex to make init_complete valid */ 2914 mutex_lock(&acpi_desc->init_mutex); 2915 if (acpi_desc->cancel || acpi_desc->init_complete) { 2916 mutex_unlock(&acpi_desc->init_mutex); 2917 return 0; 2918 } 2919 2920 /* 2921 * Scrub work could take 10s of seconds, userspace may give up so we 2922 * need to be interruptible while waiting. 2923 */ 2924 INIT_WORK_ONSTACK(&flush.work, flush_probe); 2925 init_completion(&flush.cmp); 2926 queue_work(nfit_wq, &flush.work); 2927 mutex_unlock(&acpi_desc->init_mutex); 2928 2929 rc = wait_for_completion_interruptible(&flush.cmp); 2930 cancel_work_sync(&flush.work); 2931 return rc; 2932 } 2933 2934 static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc, 2935 struct nvdimm *nvdimm, unsigned int cmd) 2936 { 2937 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); 2938 2939 if (nvdimm) 2940 return 0; 2941 if (cmd != ND_CMD_ARS_START) 2942 return 0; 2943 2944 /* 2945 * The kernel and userspace may race to initiate a scrub, but 2946 * the scrub thread is prepared to lose that initial race. It 2947 * just needs guarantees that any ars it initiates are not 2948 * interrupted by any intervening start reqeusts from userspace. 2949 */ 2950 if (work_busy(&acpi_desc->work)) 2951 return -EBUSY; 2952 2953 return 0; 2954 } 2955 2956 int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc, u8 flags) 2957 { 2958 struct device *dev = acpi_desc->dev; 2959 struct nfit_spa *nfit_spa; 2960 2961 if (work_busy(&acpi_desc->work)) 2962 return -EBUSY; 2963 2964 mutex_lock(&acpi_desc->init_mutex); 2965 if (acpi_desc->cancel) { 2966 mutex_unlock(&acpi_desc->init_mutex); 2967 return 0; 2968 } 2969 2970 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { 2971 struct acpi_nfit_system_address *spa = nfit_spa->spa; 2972 2973 if (nfit_spa_type(spa) != NFIT_SPA_PM) 2974 continue; 2975 2976 nfit_spa->ars_required = 1; 2977 } 2978 acpi_desc->ars_start_flags = flags; 2979 queue_work(nfit_wq, &acpi_desc->work); 2980 dev_dbg(dev, "%s: ars_scan triggered\n", __func__); 2981 mutex_unlock(&acpi_desc->init_mutex); 2982 2983 return 0; 2984 } 2985 2986 void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev) 2987 { 2988 struct nvdimm_bus_descriptor *nd_desc; 2989 2990 dev_set_drvdata(dev, acpi_desc); 2991 acpi_desc->dev = dev; 2992 acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io; 2993 nd_desc = &acpi_desc->nd_desc; 2994 nd_desc->provider_name = "ACPI.NFIT"; 2995 nd_desc->module = THIS_MODULE; 2996 nd_desc->ndctl = acpi_nfit_ctl; 2997 nd_desc->flush_probe = acpi_nfit_flush_probe; 2998 nd_desc->clear_to_send = acpi_nfit_clear_to_send; 2999 nd_desc->attr_groups = acpi_nfit_attribute_groups; 3000 3001 INIT_LIST_HEAD(&acpi_desc->spas); 3002 INIT_LIST_HEAD(&acpi_desc->dcrs); 3003 INIT_LIST_HEAD(&acpi_desc->bdws); 3004 INIT_LIST_HEAD(&acpi_desc->idts); 3005 INIT_LIST_HEAD(&acpi_desc->flushes); 3006 INIT_LIST_HEAD(&acpi_desc->memdevs); 3007 INIT_LIST_HEAD(&acpi_desc->dimms); 3008 INIT_LIST_HEAD(&acpi_desc->list); 3009 mutex_init(&acpi_desc->init_mutex); 3010 INIT_WORK(&acpi_desc->work, acpi_nfit_scrub); 3011 } 3012 EXPORT_SYMBOL_GPL(acpi_nfit_desc_init); 3013 3014 static void acpi_nfit_put_table(void *table) 3015 { 3016 acpi_put_table(table); 3017 } 3018 3019 void acpi_nfit_shutdown(void *data) 3020 { 3021 struct acpi_nfit_desc *acpi_desc = data; 3022 struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); 3023 3024 /* 3025 * Destruct under acpi_desc_lock so that nfit_handle_mce does not 3026 * race teardown 3027 */ 3028 mutex_lock(&acpi_desc_lock); 3029 list_del(&acpi_desc->list); 3030 mutex_unlock(&acpi_desc_lock); 3031 3032 mutex_lock(&acpi_desc->init_mutex); 3033 acpi_desc->cancel = 1; 3034 mutex_unlock(&acpi_desc->init_mutex); 3035 3036 /* 3037 * Bounce the nvdimm bus lock to make sure any in-flight 3038 * acpi_nfit_ars_rescan() submissions have had a chance to 3039 * either submit or see ->cancel set. 3040 */ 3041 device_lock(bus_dev); 3042 device_unlock(bus_dev); 3043 3044 flush_workqueue(nfit_wq); 3045 } 3046 EXPORT_SYMBOL_GPL(acpi_nfit_shutdown); 3047 3048 static int acpi_nfit_add(struct acpi_device *adev) 3049 { 3050 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 3051 struct acpi_nfit_desc *acpi_desc; 3052 struct device *dev = &adev->dev; 3053 struct acpi_table_header *tbl; 3054 acpi_status status = AE_OK; 3055 acpi_size sz; 3056 int rc = 0; 3057 3058 status = acpi_get_table(ACPI_SIG_NFIT, 0, &tbl); 3059 if (ACPI_FAILURE(status)) { 3060 /* This is ok, we could have an nvdimm hotplugged later */ 3061 dev_dbg(dev, "failed to find NFIT at startup\n"); 3062 return 0; 3063 } 3064 3065 rc = devm_add_action_or_reset(dev, acpi_nfit_put_table, tbl); 3066 if (rc) 3067 return rc; 3068 sz = tbl->length; 3069 3070 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL); 3071 if (!acpi_desc) 3072 return -ENOMEM; 3073 acpi_nfit_desc_init(acpi_desc, &adev->dev); 3074 3075 /* Save the acpi header for exporting the revision via sysfs */ 3076 acpi_desc->acpi_header = *tbl; 3077 3078 /* Evaluate _FIT and override with that if present */ 3079 status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf); 3080 if (ACPI_SUCCESS(status) && buf.length > 0) { 3081 union acpi_object *obj = buf.pointer; 3082 3083 if (obj->type == ACPI_TYPE_BUFFER) 3084 rc = acpi_nfit_init(acpi_desc, obj->buffer.pointer, 3085 obj->buffer.length); 3086 else 3087 dev_dbg(dev, "%s invalid type %d, ignoring _FIT\n", 3088 __func__, (int) obj->type); 3089 kfree(buf.pointer); 3090 } else 3091 /* skip over the lead-in header table */ 3092 rc = acpi_nfit_init(acpi_desc, (void *) tbl 3093 + sizeof(struct acpi_table_nfit), 3094 sz - sizeof(struct acpi_table_nfit)); 3095 3096 if (rc) 3097 return rc; 3098 return devm_add_action_or_reset(dev, acpi_nfit_shutdown, acpi_desc); 3099 } 3100 3101 static int acpi_nfit_remove(struct acpi_device *adev) 3102 { 3103 /* see acpi_nfit_unregister */ 3104 return 0; 3105 } 3106 3107 static void acpi_nfit_update_notify(struct device *dev, acpi_handle handle) 3108 { 3109 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev); 3110 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 3111 union acpi_object *obj; 3112 acpi_status status; 3113 int ret; 3114 3115 if (!dev->driver) { 3116 /* dev->driver may be null if we're being removed */ 3117 dev_dbg(dev, "%s: no driver found for dev\n", __func__); 3118 return; 3119 } 3120 3121 if (!acpi_desc) { 3122 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL); 3123 if (!acpi_desc) 3124 return; 3125 acpi_nfit_desc_init(acpi_desc, dev); 3126 } else { 3127 /* 3128 * Finish previous registration before considering new 3129 * regions. 3130 */ 3131 flush_workqueue(nfit_wq); 3132 } 3133 3134 /* Evaluate _FIT */ 3135 status = acpi_evaluate_object(handle, "_FIT", NULL, &buf); 3136 if (ACPI_FAILURE(status)) { 3137 dev_err(dev, "failed to evaluate _FIT\n"); 3138 return; 3139 } 3140 3141 obj = buf.pointer; 3142 if (obj->type == ACPI_TYPE_BUFFER) { 3143 ret = acpi_nfit_init(acpi_desc, obj->buffer.pointer, 3144 obj->buffer.length); 3145 if (ret) 3146 dev_err(dev, "failed to merge updated NFIT\n"); 3147 } else 3148 dev_err(dev, "Invalid _FIT\n"); 3149 kfree(buf.pointer); 3150 } 3151 3152 static void acpi_nfit_uc_error_notify(struct device *dev, acpi_handle handle) 3153 { 3154 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev); 3155 u8 flags = (acpi_desc->scrub_mode == HW_ERROR_SCRUB_ON) ? 3156 0 : ND_ARS_RETURN_PREV_DATA; 3157 3158 acpi_nfit_ars_rescan(acpi_desc, flags); 3159 } 3160 3161 void __acpi_nfit_notify(struct device *dev, acpi_handle handle, u32 event) 3162 { 3163 dev_dbg(dev, "%s: event: 0x%x\n", __func__, event); 3164 3165 switch (event) { 3166 case NFIT_NOTIFY_UPDATE: 3167 return acpi_nfit_update_notify(dev, handle); 3168 case NFIT_NOTIFY_UC_MEMORY_ERROR: 3169 return acpi_nfit_uc_error_notify(dev, handle); 3170 default: 3171 return; 3172 } 3173 } 3174 EXPORT_SYMBOL_GPL(__acpi_nfit_notify); 3175 3176 static void acpi_nfit_notify(struct acpi_device *adev, u32 event) 3177 { 3178 device_lock(&adev->dev); 3179 __acpi_nfit_notify(&adev->dev, adev->handle, event); 3180 device_unlock(&adev->dev); 3181 } 3182 3183 static const struct acpi_device_id acpi_nfit_ids[] = { 3184 { "ACPI0012", 0 }, 3185 { "", 0 }, 3186 }; 3187 MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids); 3188 3189 static struct acpi_driver acpi_nfit_driver = { 3190 .name = KBUILD_MODNAME, 3191 .ids = acpi_nfit_ids, 3192 .ops = { 3193 .add = acpi_nfit_add, 3194 .remove = acpi_nfit_remove, 3195 .notify = acpi_nfit_notify, 3196 }, 3197 }; 3198 3199 static __init int nfit_init(void) 3200 { 3201 int ret; 3202 3203 BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40); 3204 BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56); 3205 BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48); 3206 BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20); 3207 BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9); 3208 BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80); 3209 BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40); 3210 3211 guid_parse(UUID_VOLATILE_MEMORY, &nfit_uuid[NFIT_SPA_VOLATILE]); 3212 guid_parse(UUID_PERSISTENT_MEMORY, &nfit_uuid[NFIT_SPA_PM]); 3213 guid_parse(UUID_CONTROL_REGION, &nfit_uuid[NFIT_SPA_DCR]); 3214 guid_parse(UUID_DATA_REGION, &nfit_uuid[NFIT_SPA_BDW]); 3215 guid_parse(UUID_VOLATILE_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_VDISK]); 3216 guid_parse(UUID_VOLATILE_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_VCD]); 3217 guid_parse(UUID_PERSISTENT_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_PDISK]); 3218 guid_parse(UUID_PERSISTENT_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_PCD]); 3219 guid_parse(UUID_NFIT_BUS, &nfit_uuid[NFIT_DEV_BUS]); 3220 guid_parse(UUID_NFIT_DIMM, &nfit_uuid[NFIT_DEV_DIMM]); 3221 guid_parse(UUID_NFIT_DIMM_N_HPE1, &nfit_uuid[NFIT_DEV_DIMM_N_HPE1]); 3222 guid_parse(UUID_NFIT_DIMM_N_HPE2, &nfit_uuid[NFIT_DEV_DIMM_N_HPE2]); 3223 guid_parse(UUID_NFIT_DIMM_N_MSFT, &nfit_uuid[NFIT_DEV_DIMM_N_MSFT]); 3224 3225 nfit_wq = create_singlethread_workqueue("nfit"); 3226 if (!nfit_wq) 3227 return -ENOMEM; 3228 3229 nfit_mce_register(); 3230 ret = acpi_bus_register_driver(&acpi_nfit_driver); 3231 if (ret) { 3232 nfit_mce_unregister(); 3233 destroy_workqueue(nfit_wq); 3234 } 3235 3236 return ret; 3237 3238 } 3239 3240 static __exit void nfit_exit(void) 3241 { 3242 nfit_mce_unregister(); 3243 acpi_bus_unregister_driver(&acpi_nfit_driver); 3244 destroy_workqueue(nfit_wq); 3245 WARN_ON(!list_empty(&acpi_descs)); 3246 } 3247 3248 module_init(nfit_init); 3249 module_exit(nfit_exit); 3250 MODULE_LICENSE("GPL v2"); 3251 MODULE_AUTHOR("Intel Corporation"); 3252