1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved. 4 */ 5 #include <linux/scatterlist.h> 6 #include <linux/highmem.h> 7 #include <linux/sched.h> 8 #include <linux/slab.h> 9 #include <linux/hash.h> 10 #include <linux/sort.h> 11 #include <linux/io.h> 12 #include <linux/nd.h> 13 #include "nd-core.h" 14 #include "nd.h" 15 16 /* 17 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is 18 * irrelevant. 19 */ 20 #include <linux/io-64-nonatomic-hi-lo.h> 21 22 static DEFINE_IDA(region_ida); 23 static DEFINE_PER_CPU(int, flush_idx); 24 25 static int nvdimm_map_flush(struct device *dev, struct nvdimm *nvdimm, int dimm, 26 struct nd_region_data *ndrd) 27 { 28 int i, j; 29 30 dev_dbg(dev, "%s: map %d flush address%s\n", nvdimm_name(nvdimm), 31 nvdimm->num_flush, nvdimm->num_flush == 1 ? "" : "es"); 32 for (i = 0; i < (1 << ndrd->hints_shift); i++) { 33 struct resource *res = &nvdimm->flush_wpq[i]; 34 unsigned long pfn = PHYS_PFN(res->start); 35 void __iomem *flush_page; 36 37 /* check if flush hints share a page */ 38 for (j = 0; j < i; j++) { 39 struct resource *res_j = &nvdimm->flush_wpq[j]; 40 unsigned long pfn_j = PHYS_PFN(res_j->start); 41 42 if (pfn == pfn_j) 43 break; 44 } 45 46 if (j < i) 47 flush_page = (void __iomem *) ((unsigned long) 48 ndrd_get_flush_wpq(ndrd, dimm, j) 49 & PAGE_MASK); 50 else 51 flush_page = devm_nvdimm_ioremap(dev, 52 PFN_PHYS(pfn), PAGE_SIZE); 53 if (!flush_page) 54 return -ENXIO; 55 ndrd_set_flush_wpq(ndrd, dimm, i, flush_page 56 + (res->start & ~PAGE_MASK)); 57 } 58 59 return 0; 60 } 61 62 int nd_region_activate(struct nd_region *nd_region) 63 { 64 int i, j, num_flush = 0; 65 struct nd_region_data *ndrd; 66 struct device *dev = &nd_region->dev; 67 size_t flush_data_size = sizeof(void *); 68 69 nvdimm_bus_lock(&nd_region->dev); 70 for (i = 0; i < nd_region->ndr_mappings; i++) { 71 struct nd_mapping *nd_mapping = &nd_region->mapping[i]; 72 struct nvdimm *nvdimm = nd_mapping->nvdimm; 73 74 if (test_bit(NDD_SECURITY_OVERWRITE, &nvdimm->flags)) { 75 nvdimm_bus_unlock(&nd_region->dev); 76 return -EBUSY; 77 } 78 79 /* at least one null hint slot per-dimm for the "no-hint" case */ 80 flush_data_size += sizeof(void *); 81 num_flush = min_not_zero(num_flush, nvdimm->num_flush); 82 if (!nvdimm->num_flush) 83 continue; 84 flush_data_size += nvdimm->num_flush * sizeof(void *); 85 } 86 nvdimm_bus_unlock(&nd_region->dev); 87 88 ndrd = devm_kzalloc(dev, sizeof(*ndrd) + flush_data_size, GFP_KERNEL); 89 if (!ndrd) 90 return -ENOMEM; 91 dev_set_drvdata(dev, ndrd); 92 93 if (!num_flush) 94 return 0; 95 96 ndrd->hints_shift = ilog2(num_flush); 97 for (i = 0; i < nd_region->ndr_mappings; i++) { 98 struct nd_mapping *nd_mapping = &nd_region->mapping[i]; 99 struct nvdimm *nvdimm = nd_mapping->nvdimm; 100 int rc = nvdimm_map_flush(&nd_region->dev, nvdimm, i, ndrd); 101 102 if (rc) 103 return rc; 104 } 105 106 /* 107 * Clear out entries that are duplicates. This should prevent the 108 * extra flushings. 109 */ 110 for (i = 0; i < nd_region->ndr_mappings - 1; i++) { 111 /* ignore if NULL already */ 112 if (!ndrd_get_flush_wpq(ndrd, i, 0)) 113 continue; 114 115 for (j = i + 1; j < nd_region->ndr_mappings; j++) 116 if (ndrd_get_flush_wpq(ndrd, i, 0) == 117 ndrd_get_flush_wpq(ndrd, j, 0)) 118 ndrd_set_flush_wpq(ndrd, j, 0, NULL); 119 } 120 121 return 0; 122 } 123 124 static void nd_region_release(struct device *dev) 125 { 126 struct nd_region *nd_region = to_nd_region(dev); 127 u16 i; 128 129 for (i = 0; i < nd_region->ndr_mappings; i++) { 130 struct nd_mapping *nd_mapping = &nd_region->mapping[i]; 131 struct nvdimm *nvdimm = nd_mapping->nvdimm; 132 133 put_device(&nvdimm->dev); 134 } 135 free_percpu(nd_region->lane); 136 ida_simple_remove(®ion_ida, nd_region->id); 137 if (is_nd_blk(dev)) 138 kfree(to_nd_blk_region(dev)); 139 else 140 kfree(nd_region); 141 } 142 143 struct nd_region *to_nd_region(struct device *dev) 144 { 145 struct nd_region *nd_region = container_of(dev, struct nd_region, dev); 146 147 WARN_ON(dev->type->release != nd_region_release); 148 return nd_region; 149 } 150 EXPORT_SYMBOL_GPL(to_nd_region); 151 152 struct device *nd_region_dev(struct nd_region *nd_region) 153 { 154 if (!nd_region) 155 return NULL; 156 return &nd_region->dev; 157 } 158 EXPORT_SYMBOL_GPL(nd_region_dev); 159 160 struct nd_blk_region *to_nd_blk_region(struct device *dev) 161 { 162 struct nd_region *nd_region = to_nd_region(dev); 163 164 WARN_ON(!is_nd_blk(dev)); 165 return container_of(nd_region, struct nd_blk_region, nd_region); 166 } 167 EXPORT_SYMBOL_GPL(to_nd_blk_region); 168 169 void *nd_region_provider_data(struct nd_region *nd_region) 170 { 171 return nd_region->provider_data; 172 } 173 EXPORT_SYMBOL_GPL(nd_region_provider_data); 174 175 void *nd_blk_region_provider_data(struct nd_blk_region *ndbr) 176 { 177 return ndbr->blk_provider_data; 178 } 179 EXPORT_SYMBOL_GPL(nd_blk_region_provider_data); 180 181 void nd_blk_region_set_provider_data(struct nd_blk_region *ndbr, void *data) 182 { 183 ndbr->blk_provider_data = data; 184 } 185 EXPORT_SYMBOL_GPL(nd_blk_region_set_provider_data); 186 187 /** 188 * nd_region_to_nstype() - region to an integer namespace type 189 * @nd_region: region-device to interrogate 190 * 191 * This is the 'nstype' attribute of a region as well, an input to the 192 * MODALIAS for namespace devices, and bit number for a nvdimm_bus to match 193 * namespace devices with namespace drivers. 194 */ 195 int nd_region_to_nstype(struct nd_region *nd_region) 196 { 197 if (is_memory(&nd_region->dev)) { 198 u16 i, alias; 199 200 for (i = 0, alias = 0; i < nd_region->ndr_mappings; i++) { 201 struct nd_mapping *nd_mapping = &nd_region->mapping[i]; 202 struct nvdimm *nvdimm = nd_mapping->nvdimm; 203 204 if (test_bit(NDD_ALIASING, &nvdimm->flags)) 205 alias++; 206 } 207 if (alias) 208 return ND_DEVICE_NAMESPACE_PMEM; 209 else 210 return ND_DEVICE_NAMESPACE_IO; 211 } else if (is_nd_blk(&nd_region->dev)) { 212 return ND_DEVICE_NAMESPACE_BLK; 213 } 214 215 return 0; 216 } 217 EXPORT_SYMBOL(nd_region_to_nstype); 218 219 static ssize_t size_show(struct device *dev, 220 struct device_attribute *attr, char *buf) 221 { 222 struct nd_region *nd_region = to_nd_region(dev); 223 unsigned long long size = 0; 224 225 if (is_memory(dev)) { 226 size = nd_region->ndr_size; 227 } else if (nd_region->ndr_mappings == 1) { 228 struct nd_mapping *nd_mapping = &nd_region->mapping[0]; 229 230 size = nd_mapping->size; 231 } 232 233 return sprintf(buf, "%llu\n", size); 234 } 235 static DEVICE_ATTR_RO(size); 236 237 static ssize_t deep_flush_show(struct device *dev, 238 struct device_attribute *attr, char *buf) 239 { 240 struct nd_region *nd_region = to_nd_region(dev); 241 242 /* 243 * NOTE: in the nvdimm_has_flush() error case this attribute is 244 * not visible. 245 */ 246 return sprintf(buf, "%d\n", nvdimm_has_flush(nd_region)); 247 } 248 249 static ssize_t deep_flush_store(struct device *dev, struct device_attribute *attr, 250 const char *buf, size_t len) 251 { 252 bool flush; 253 int rc = strtobool(buf, &flush); 254 struct nd_region *nd_region = to_nd_region(dev); 255 256 if (rc) 257 return rc; 258 if (!flush) 259 return -EINVAL; 260 rc = nvdimm_flush(nd_region, NULL); 261 if (rc) 262 return rc; 263 264 return len; 265 } 266 static DEVICE_ATTR_RW(deep_flush); 267 268 static ssize_t mappings_show(struct device *dev, 269 struct device_attribute *attr, char *buf) 270 { 271 struct nd_region *nd_region = to_nd_region(dev); 272 273 return sprintf(buf, "%d\n", nd_region->ndr_mappings); 274 } 275 static DEVICE_ATTR_RO(mappings); 276 277 static ssize_t nstype_show(struct device *dev, 278 struct device_attribute *attr, char *buf) 279 { 280 struct nd_region *nd_region = to_nd_region(dev); 281 282 return sprintf(buf, "%d\n", nd_region_to_nstype(nd_region)); 283 } 284 static DEVICE_ATTR_RO(nstype); 285 286 static ssize_t set_cookie_show(struct device *dev, 287 struct device_attribute *attr, char *buf) 288 { 289 struct nd_region *nd_region = to_nd_region(dev); 290 struct nd_interleave_set *nd_set = nd_region->nd_set; 291 ssize_t rc = 0; 292 293 if (is_memory(dev) && nd_set) 294 /* pass, should be precluded by region_visible */; 295 else 296 return -ENXIO; 297 298 /* 299 * The cookie to show depends on which specification of the 300 * labels we are using. If there are not labels then default to 301 * the v1.1 namespace label cookie definition. To read all this 302 * data we need to wait for probing to settle. 303 */ 304 nd_device_lock(dev); 305 nvdimm_bus_lock(dev); 306 wait_nvdimm_bus_probe_idle(dev); 307 if (nd_region->ndr_mappings) { 308 struct nd_mapping *nd_mapping = &nd_region->mapping[0]; 309 struct nvdimm_drvdata *ndd = to_ndd(nd_mapping); 310 311 if (ndd) { 312 struct nd_namespace_index *nsindex; 313 314 nsindex = to_namespace_index(ndd, ndd->ns_current); 315 rc = sprintf(buf, "%#llx\n", 316 nd_region_interleave_set_cookie(nd_region, 317 nsindex)); 318 } 319 } 320 nvdimm_bus_unlock(dev); 321 nd_device_unlock(dev); 322 323 if (rc) 324 return rc; 325 return sprintf(buf, "%#llx\n", nd_set->cookie1); 326 } 327 static DEVICE_ATTR_RO(set_cookie); 328 329 resource_size_t nd_region_available_dpa(struct nd_region *nd_region) 330 { 331 resource_size_t blk_max_overlap = 0, available, overlap; 332 int i; 333 334 WARN_ON(!is_nvdimm_bus_locked(&nd_region->dev)); 335 336 retry: 337 available = 0; 338 overlap = blk_max_overlap; 339 for (i = 0; i < nd_region->ndr_mappings; i++) { 340 struct nd_mapping *nd_mapping = &nd_region->mapping[i]; 341 struct nvdimm_drvdata *ndd = to_ndd(nd_mapping); 342 343 /* if a dimm is disabled the available capacity is zero */ 344 if (!ndd) 345 return 0; 346 347 if (is_memory(&nd_region->dev)) { 348 available += nd_pmem_available_dpa(nd_region, 349 nd_mapping, &overlap); 350 if (overlap > blk_max_overlap) { 351 blk_max_overlap = overlap; 352 goto retry; 353 } 354 } else if (is_nd_blk(&nd_region->dev)) 355 available += nd_blk_available_dpa(nd_region); 356 } 357 358 return available; 359 } 360 361 resource_size_t nd_region_allocatable_dpa(struct nd_region *nd_region) 362 { 363 resource_size_t available = 0; 364 int i; 365 366 if (is_memory(&nd_region->dev)) 367 available = PHYS_ADDR_MAX; 368 369 WARN_ON(!is_nvdimm_bus_locked(&nd_region->dev)); 370 for (i = 0; i < nd_region->ndr_mappings; i++) { 371 struct nd_mapping *nd_mapping = &nd_region->mapping[i]; 372 373 if (is_memory(&nd_region->dev)) 374 available = min(available, 375 nd_pmem_max_contiguous_dpa(nd_region, 376 nd_mapping)); 377 else if (is_nd_blk(&nd_region->dev)) 378 available += nd_blk_available_dpa(nd_region); 379 } 380 if (is_memory(&nd_region->dev)) 381 return available * nd_region->ndr_mappings; 382 return available; 383 } 384 385 static ssize_t available_size_show(struct device *dev, 386 struct device_attribute *attr, char *buf) 387 { 388 struct nd_region *nd_region = to_nd_region(dev); 389 unsigned long long available = 0; 390 391 /* 392 * Flush in-flight updates and grab a snapshot of the available 393 * size. Of course, this value is potentially invalidated the 394 * memory nvdimm_bus_lock() is dropped, but that's userspace's 395 * problem to not race itself. 396 */ 397 nd_device_lock(dev); 398 nvdimm_bus_lock(dev); 399 wait_nvdimm_bus_probe_idle(dev); 400 available = nd_region_available_dpa(nd_region); 401 nvdimm_bus_unlock(dev); 402 nd_device_unlock(dev); 403 404 return sprintf(buf, "%llu\n", available); 405 } 406 static DEVICE_ATTR_RO(available_size); 407 408 static ssize_t max_available_extent_show(struct device *dev, 409 struct device_attribute *attr, char *buf) 410 { 411 struct nd_region *nd_region = to_nd_region(dev); 412 unsigned long long available = 0; 413 414 nd_device_lock(dev); 415 nvdimm_bus_lock(dev); 416 wait_nvdimm_bus_probe_idle(dev); 417 available = nd_region_allocatable_dpa(nd_region); 418 nvdimm_bus_unlock(dev); 419 nd_device_unlock(dev); 420 421 return sprintf(buf, "%llu\n", available); 422 } 423 static DEVICE_ATTR_RO(max_available_extent); 424 425 static ssize_t init_namespaces_show(struct device *dev, 426 struct device_attribute *attr, char *buf) 427 { 428 struct nd_region_data *ndrd = dev_get_drvdata(dev); 429 ssize_t rc; 430 431 nvdimm_bus_lock(dev); 432 if (ndrd) 433 rc = sprintf(buf, "%d/%d\n", ndrd->ns_active, ndrd->ns_count); 434 else 435 rc = -ENXIO; 436 nvdimm_bus_unlock(dev); 437 438 return rc; 439 } 440 static DEVICE_ATTR_RO(init_namespaces); 441 442 static ssize_t namespace_seed_show(struct device *dev, 443 struct device_attribute *attr, char *buf) 444 { 445 struct nd_region *nd_region = to_nd_region(dev); 446 ssize_t rc; 447 448 nvdimm_bus_lock(dev); 449 if (nd_region->ns_seed) 450 rc = sprintf(buf, "%s\n", dev_name(nd_region->ns_seed)); 451 else 452 rc = sprintf(buf, "\n"); 453 nvdimm_bus_unlock(dev); 454 return rc; 455 } 456 static DEVICE_ATTR_RO(namespace_seed); 457 458 static ssize_t btt_seed_show(struct device *dev, 459 struct device_attribute *attr, char *buf) 460 { 461 struct nd_region *nd_region = to_nd_region(dev); 462 ssize_t rc; 463 464 nvdimm_bus_lock(dev); 465 if (nd_region->btt_seed) 466 rc = sprintf(buf, "%s\n", dev_name(nd_region->btt_seed)); 467 else 468 rc = sprintf(buf, "\n"); 469 nvdimm_bus_unlock(dev); 470 471 return rc; 472 } 473 static DEVICE_ATTR_RO(btt_seed); 474 475 static ssize_t pfn_seed_show(struct device *dev, 476 struct device_attribute *attr, char *buf) 477 { 478 struct nd_region *nd_region = to_nd_region(dev); 479 ssize_t rc; 480 481 nvdimm_bus_lock(dev); 482 if (nd_region->pfn_seed) 483 rc = sprintf(buf, "%s\n", dev_name(nd_region->pfn_seed)); 484 else 485 rc = sprintf(buf, "\n"); 486 nvdimm_bus_unlock(dev); 487 488 return rc; 489 } 490 static DEVICE_ATTR_RO(pfn_seed); 491 492 static ssize_t dax_seed_show(struct device *dev, 493 struct device_attribute *attr, char *buf) 494 { 495 struct nd_region *nd_region = to_nd_region(dev); 496 ssize_t rc; 497 498 nvdimm_bus_lock(dev); 499 if (nd_region->dax_seed) 500 rc = sprintf(buf, "%s\n", dev_name(nd_region->dax_seed)); 501 else 502 rc = sprintf(buf, "\n"); 503 nvdimm_bus_unlock(dev); 504 505 return rc; 506 } 507 static DEVICE_ATTR_RO(dax_seed); 508 509 static ssize_t read_only_show(struct device *dev, 510 struct device_attribute *attr, char *buf) 511 { 512 struct nd_region *nd_region = to_nd_region(dev); 513 514 return sprintf(buf, "%d\n", nd_region->ro); 515 } 516 517 static ssize_t read_only_store(struct device *dev, 518 struct device_attribute *attr, const char *buf, size_t len) 519 { 520 bool ro; 521 int rc = strtobool(buf, &ro); 522 struct nd_region *nd_region = to_nd_region(dev); 523 524 if (rc) 525 return rc; 526 527 nd_region->ro = ro; 528 return len; 529 } 530 static DEVICE_ATTR_RW(read_only); 531 532 static ssize_t region_badblocks_show(struct device *dev, 533 struct device_attribute *attr, char *buf) 534 { 535 struct nd_region *nd_region = to_nd_region(dev); 536 ssize_t rc; 537 538 nd_device_lock(dev); 539 if (dev->driver) 540 rc = badblocks_show(&nd_region->bb, buf, 0); 541 else 542 rc = -ENXIO; 543 nd_device_unlock(dev); 544 545 return rc; 546 } 547 static DEVICE_ATTR(badblocks, 0444, region_badblocks_show, NULL); 548 549 static ssize_t resource_show(struct device *dev, 550 struct device_attribute *attr, char *buf) 551 { 552 struct nd_region *nd_region = to_nd_region(dev); 553 554 return sprintf(buf, "%#llx\n", nd_region->ndr_start); 555 } 556 static DEVICE_ATTR_RO(resource); 557 558 static ssize_t persistence_domain_show(struct device *dev, 559 struct device_attribute *attr, char *buf) 560 { 561 struct nd_region *nd_region = to_nd_region(dev); 562 563 if (test_bit(ND_REGION_PERSIST_CACHE, &nd_region->flags)) 564 return sprintf(buf, "cpu_cache\n"); 565 else if (test_bit(ND_REGION_PERSIST_MEMCTRL, &nd_region->flags)) 566 return sprintf(buf, "memory_controller\n"); 567 else 568 return sprintf(buf, "\n"); 569 } 570 static DEVICE_ATTR_RO(persistence_domain); 571 572 static struct attribute *nd_region_attributes[] = { 573 &dev_attr_size.attr, 574 &dev_attr_nstype.attr, 575 &dev_attr_mappings.attr, 576 &dev_attr_btt_seed.attr, 577 &dev_attr_pfn_seed.attr, 578 &dev_attr_dax_seed.attr, 579 &dev_attr_deep_flush.attr, 580 &dev_attr_read_only.attr, 581 &dev_attr_set_cookie.attr, 582 &dev_attr_available_size.attr, 583 &dev_attr_max_available_extent.attr, 584 &dev_attr_namespace_seed.attr, 585 &dev_attr_init_namespaces.attr, 586 &dev_attr_badblocks.attr, 587 &dev_attr_resource.attr, 588 &dev_attr_persistence_domain.attr, 589 NULL, 590 }; 591 592 static umode_t region_visible(struct kobject *kobj, struct attribute *a, int n) 593 { 594 struct device *dev = container_of(kobj, typeof(*dev), kobj); 595 struct nd_region *nd_region = to_nd_region(dev); 596 struct nd_interleave_set *nd_set = nd_region->nd_set; 597 int type = nd_region_to_nstype(nd_region); 598 599 if (!is_memory(dev) && a == &dev_attr_pfn_seed.attr) 600 return 0; 601 602 if (!is_memory(dev) && a == &dev_attr_dax_seed.attr) 603 return 0; 604 605 if (!is_memory(dev) && a == &dev_attr_badblocks.attr) 606 return 0; 607 608 if (a == &dev_attr_resource.attr) { 609 if (is_memory(dev)) 610 return 0400; 611 else 612 return 0; 613 } 614 615 if (a == &dev_attr_deep_flush.attr) { 616 int has_flush = nvdimm_has_flush(nd_region); 617 618 if (has_flush == 1) 619 return a->mode; 620 else if (has_flush == 0) 621 return 0444; 622 else 623 return 0; 624 } 625 626 if (a == &dev_attr_persistence_domain.attr) { 627 if ((nd_region->flags & (BIT(ND_REGION_PERSIST_CACHE) 628 | BIT(ND_REGION_PERSIST_MEMCTRL))) == 0) 629 return 0; 630 return a->mode; 631 } 632 633 if (a != &dev_attr_set_cookie.attr 634 && a != &dev_attr_available_size.attr) 635 return a->mode; 636 637 if ((type == ND_DEVICE_NAMESPACE_PMEM 638 || type == ND_DEVICE_NAMESPACE_BLK) 639 && a == &dev_attr_available_size.attr) 640 return a->mode; 641 else if (is_memory(dev) && nd_set) 642 return a->mode; 643 644 return 0; 645 } 646 647 static ssize_t mappingN(struct device *dev, char *buf, int n) 648 { 649 struct nd_region *nd_region = to_nd_region(dev); 650 struct nd_mapping *nd_mapping; 651 struct nvdimm *nvdimm; 652 653 if (n >= nd_region->ndr_mappings) 654 return -ENXIO; 655 nd_mapping = &nd_region->mapping[n]; 656 nvdimm = nd_mapping->nvdimm; 657 658 return sprintf(buf, "%s,%llu,%llu,%d\n", dev_name(&nvdimm->dev), 659 nd_mapping->start, nd_mapping->size, 660 nd_mapping->position); 661 } 662 663 #define REGION_MAPPING(idx) \ 664 static ssize_t mapping##idx##_show(struct device *dev, \ 665 struct device_attribute *attr, char *buf) \ 666 { \ 667 return mappingN(dev, buf, idx); \ 668 } \ 669 static DEVICE_ATTR_RO(mapping##idx) 670 671 /* 672 * 32 should be enough for a while, even in the presence of socket 673 * interleave a 32-way interleave set is a degenerate case. 674 */ 675 REGION_MAPPING(0); 676 REGION_MAPPING(1); 677 REGION_MAPPING(2); 678 REGION_MAPPING(3); 679 REGION_MAPPING(4); 680 REGION_MAPPING(5); 681 REGION_MAPPING(6); 682 REGION_MAPPING(7); 683 REGION_MAPPING(8); 684 REGION_MAPPING(9); 685 REGION_MAPPING(10); 686 REGION_MAPPING(11); 687 REGION_MAPPING(12); 688 REGION_MAPPING(13); 689 REGION_MAPPING(14); 690 REGION_MAPPING(15); 691 REGION_MAPPING(16); 692 REGION_MAPPING(17); 693 REGION_MAPPING(18); 694 REGION_MAPPING(19); 695 REGION_MAPPING(20); 696 REGION_MAPPING(21); 697 REGION_MAPPING(22); 698 REGION_MAPPING(23); 699 REGION_MAPPING(24); 700 REGION_MAPPING(25); 701 REGION_MAPPING(26); 702 REGION_MAPPING(27); 703 REGION_MAPPING(28); 704 REGION_MAPPING(29); 705 REGION_MAPPING(30); 706 REGION_MAPPING(31); 707 708 static umode_t mapping_visible(struct kobject *kobj, struct attribute *a, int n) 709 { 710 struct device *dev = container_of(kobj, struct device, kobj); 711 struct nd_region *nd_region = to_nd_region(dev); 712 713 if (n < nd_region->ndr_mappings) 714 return a->mode; 715 return 0; 716 } 717 718 static struct attribute *mapping_attributes[] = { 719 &dev_attr_mapping0.attr, 720 &dev_attr_mapping1.attr, 721 &dev_attr_mapping2.attr, 722 &dev_attr_mapping3.attr, 723 &dev_attr_mapping4.attr, 724 &dev_attr_mapping5.attr, 725 &dev_attr_mapping6.attr, 726 &dev_attr_mapping7.attr, 727 &dev_attr_mapping8.attr, 728 &dev_attr_mapping9.attr, 729 &dev_attr_mapping10.attr, 730 &dev_attr_mapping11.attr, 731 &dev_attr_mapping12.attr, 732 &dev_attr_mapping13.attr, 733 &dev_attr_mapping14.attr, 734 &dev_attr_mapping15.attr, 735 &dev_attr_mapping16.attr, 736 &dev_attr_mapping17.attr, 737 &dev_attr_mapping18.attr, 738 &dev_attr_mapping19.attr, 739 &dev_attr_mapping20.attr, 740 &dev_attr_mapping21.attr, 741 &dev_attr_mapping22.attr, 742 &dev_attr_mapping23.attr, 743 &dev_attr_mapping24.attr, 744 &dev_attr_mapping25.attr, 745 &dev_attr_mapping26.attr, 746 &dev_attr_mapping27.attr, 747 &dev_attr_mapping28.attr, 748 &dev_attr_mapping29.attr, 749 &dev_attr_mapping30.attr, 750 &dev_attr_mapping31.attr, 751 NULL, 752 }; 753 754 static const struct attribute_group nd_mapping_attribute_group = { 755 .is_visible = mapping_visible, 756 .attrs = mapping_attributes, 757 }; 758 759 static const struct attribute_group nd_region_attribute_group = { 760 .attrs = nd_region_attributes, 761 .is_visible = region_visible, 762 }; 763 764 static const struct attribute_group *nd_region_attribute_groups[] = { 765 &nd_device_attribute_group, 766 &nd_region_attribute_group, 767 &nd_numa_attribute_group, 768 &nd_mapping_attribute_group, 769 NULL, 770 }; 771 772 static const struct device_type nd_blk_device_type = { 773 .name = "nd_blk", 774 .release = nd_region_release, 775 .groups = nd_region_attribute_groups, 776 }; 777 778 static const struct device_type nd_pmem_device_type = { 779 .name = "nd_pmem", 780 .release = nd_region_release, 781 .groups = nd_region_attribute_groups, 782 }; 783 784 static const struct device_type nd_volatile_device_type = { 785 .name = "nd_volatile", 786 .release = nd_region_release, 787 .groups = nd_region_attribute_groups, 788 }; 789 790 bool is_nd_pmem(struct device *dev) 791 { 792 return dev ? dev->type == &nd_pmem_device_type : false; 793 } 794 795 bool is_nd_blk(struct device *dev) 796 { 797 return dev ? dev->type == &nd_blk_device_type : false; 798 } 799 800 bool is_nd_volatile(struct device *dev) 801 { 802 return dev ? dev->type == &nd_volatile_device_type : false; 803 } 804 805 u64 nd_region_interleave_set_cookie(struct nd_region *nd_region, 806 struct nd_namespace_index *nsindex) 807 { 808 struct nd_interleave_set *nd_set = nd_region->nd_set; 809 810 if (!nd_set) 811 return 0; 812 813 if (nsindex && __le16_to_cpu(nsindex->major) == 1 814 && __le16_to_cpu(nsindex->minor) == 1) 815 return nd_set->cookie1; 816 return nd_set->cookie2; 817 } 818 819 u64 nd_region_interleave_set_altcookie(struct nd_region *nd_region) 820 { 821 struct nd_interleave_set *nd_set = nd_region->nd_set; 822 823 if (nd_set) 824 return nd_set->altcookie; 825 return 0; 826 } 827 828 void nd_mapping_free_labels(struct nd_mapping *nd_mapping) 829 { 830 struct nd_label_ent *label_ent, *e; 831 832 lockdep_assert_held(&nd_mapping->lock); 833 list_for_each_entry_safe(label_ent, e, &nd_mapping->labels, list) { 834 list_del(&label_ent->list); 835 kfree(label_ent); 836 } 837 } 838 839 /* 840 * When a namespace is activated create new seeds for the next 841 * namespace, or namespace-personality to be configured. 842 */ 843 void nd_region_advance_seeds(struct nd_region *nd_region, struct device *dev) 844 { 845 nvdimm_bus_lock(dev); 846 if (nd_region->ns_seed == dev) { 847 nd_region_create_ns_seed(nd_region); 848 } else if (is_nd_btt(dev)) { 849 struct nd_btt *nd_btt = to_nd_btt(dev); 850 851 if (nd_region->btt_seed == dev) 852 nd_region_create_btt_seed(nd_region); 853 if (nd_region->ns_seed == &nd_btt->ndns->dev) 854 nd_region_create_ns_seed(nd_region); 855 } else if (is_nd_pfn(dev)) { 856 struct nd_pfn *nd_pfn = to_nd_pfn(dev); 857 858 if (nd_region->pfn_seed == dev) 859 nd_region_create_pfn_seed(nd_region); 860 if (nd_region->ns_seed == &nd_pfn->ndns->dev) 861 nd_region_create_ns_seed(nd_region); 862 } else if (is_nd_dax(dev)) { 863 struct nd_dax *nd_dax = to_nd_dax(dev); 864 865 if (nd_region->dax_seed == dev) 866 nd_region_create_dax_seed(nd_region); 867 if (nd_region->ns_seed == &nd_dax->nd_pfn.ndns->dev) 868 nd_region_create_ns_seed(nd_region); 869 } 870 nvdimm_bus_unlock(dev); 871 } 872 873 int nd_blk_region_init(struct nd_region *nd_region) 874 { 875 struct device *dev = &nd_region->dev; 876 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); 877 878 if (!is_nd_blk(dev)) 879 return 0; 880 881 if (nd_region->ndr_mappings < 1) { 882 dev_dbg(dev, "invalid BLK region\n"); 883 return -ENXIO; 884 } 885 886 return to_nd_blk_region(dev)->enable(nvdimm_bus, dev); 887 } 888 889 /** 890 * nd_region_acquire_lane - allocate and lock a lane 891 * @nd_region: region id and number of lanes possible 892 * 893 * A lane correlates to a BLK-data-window and/or a log slot in the BTT. 894 * We optimize for the common case where there are 256 lanes, one 895 * per-cpu. For larger systems we need to lock to share lanes. For now 896 * this implementation assumes the cost of maintaining an allocator for 897 * free lanes is on the order of the lock hold time, so it implements a 898 * static lane = cpu % num_lanes mapping. 899 * 900 * In the case of a BTT instance on top of a BLK namespace a lane may be 901 * acquired recursively. We lock on the first instance. 902 * 903 * In the case of a BTT instance on top of PMEM, we only acquire a lane 904 * for the BTT metadata updates. 905 */ 906 unsigned int nd_region_acquire_lane(struct nd_region *nd_region) 907 { 908 unsigned int cpu, lane; 909 910 cpu = get_cpu(); 911 if (nd_region->num_lanes < nr_cpu_ids) { 912 struct nd_percpu_lane *ndl_lock, *ndl_count; 913 914 lane = cpu % nd_region->num_lanes; 915 ndl_count = per_cpu_ptr(nd_region->lane, cpu); 916 ndl_lock = per_cpu_ptr(nd_region->lane, lane); 917 if (ndl_count->count++ == 0) 918 spin_lock(&ndl_lock->lock); 919 } else 920 lane = cpu; 921 922 return lane; 923 } 924 EXPORT_SYMBOL(nd_region_acquire_lane); 925 926 void nd_region_release_lane(struct nd_region *nd_region, unsigned int lane) 927 { 928 if (nd_region->num_lanes < nr_cpu_ids) { 929 unsigned int cpu = get_cpu(); 930 struct nd_percpu_lane *ndl_lock, *ndl_count; 931 932 ndl_count = per_cpu_ptr(nd_region->lane, cpu); 933 ndl_lock = per_cpu_ptr(nd_region->lane, lane); 934 if (--ndl_count->count == 0) 935 spin_unlock(&ndl_lock->lock); 936 put_cpu(); 937 } 938 put_cpu(); 939 } 940 EXPORT_SYMBOL(nd_region_release_lane); 941 942 static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus, 943 struct nd_region_desc *ndr_desc, 944 const struct device_type *dev_type, const char *caller) 945 { 946 struct nd_region *nd_region; 947 struct device *dev; 948 void *region_buf; 949 unsigned int i; 950 int ro = 0; 951 952 for (i = 0; i < ndr_desc->num_mappings; i++) { 953 struct nd_mapping_desc *mapping = &ndr_desc->mapping[i]; 954 struct nvdimm *nvdimm = mapping->nvdimm; 955 956 if ((mapping->start | mapping->size) % PAGE_SIZE) { 957 dev_err(&nvdimm_bus->dev, 958 "%s: %s mapping%d is not %ld aligned\n", 959 caller, dev_name(&nvdimm->dev), i, PAGE_SIZE); 960 return NULL; 961 } 962 963 if (test_bit(NDD_UNARMED, &nvdimm->flags)) 964 ro = 1; 965 966 if (test_bit(NDD_NOBLK, &nvdimm->flags) 967 && dev_type == &nd_blk_device_type) { 968 dev_err(&nvdimm_bus->dev, "%s: %s mapping%d is not BLK capable\n", 969 caller, dev_name(&nvdimm->dev), i); 970 return NULL; 971 } 972 } 973 974 if (dev_type == &nd_blk_device_type) { 975 struct nd_blk_region_desc *ndbr_desc; 976 struct nd_blk_region *ndbr; 977 978 ndbr_desc = to_blk_region_desc(ndr_desc); 979 ndbr = kzalloc(sizeof(*ndbr) + sizeof(struct nd_mapping) 980 * ndr_desc->num_mappings, 981 GFP_KERNEL); 982 if (ndbr) { 983 nd_region = &ndbr->nd_region; 984 ndbr->enable = ndbr_desc->enable; 985 ndbr->do_io = ndbr_desc->do_io; 986 } 987 region_buf = ndbr; 988 } else { 989 nd_region = kzalloc(struct_size(nd_region, mapping, 990 ndr_desc->num_mappings), 991 GFP_KERNEL); 992 region_buf = nd_region; 993 } 994 995 if (!region_buf) 996 return NULL; 997 nd_region->id = ida_simple_get(®ion_ida, 0, 0, GFP_KERNEL); 998 if (nd_region->id < 0) 999 goto err_id; 1000 1001 nd_region->lane = alloc_percpu(struct nd_percpu_lane); 1002 if (!nd_region->lane) 1003 goto err_percpu; 1004 1005 for (i = 0; i < nr_cpu_ids; i++) { 1006 struct nd_percpu_lane *ndl; 1007 1008 ndl = per_cpu_ptr(nd_region->lane, i); 1009 spin_lock_init(&ndl->lock); 1010 ndl->count = 0; 1011 } 1012 1013 for (i = 0; i < ndr_desc->num_mappings; i++) { 1014 struct nd_mapping_desc *mapping = &ndr_desc->mapping[i]; 1015 struct nvdimm *nvdimm = mapping->nvdimm; 1016 1017 nd_region->mapping[i].nvdimm = nvdimm; 1018 nd_region->mapping[i].start = mapping->start; 1019 nd_region->mapping[i].size = mapping->size; 1020 nd_region->mapping[i].position = mapping->position; 1021 INIT_LIST_HEAD(&nd_region->mapping[i].labels); 1022 mutex_init(&nd_region->mapping[i].lock); 1023 1024 get_device(&nvdimm->dev); 1025 } 1026 nd_region->ndr_mappings = ndr_desc->num_mappings; 1027 nd_region->provider_data = ndr_desc->provider_data; 1028 nd_region->nd_set = ndr_desc->nd_set; 1029 nd_region->num_lanes = ndr_desc->num_lanes; 1030 nd_region->flags = ndr_desc->flags; 1031 nd_region->ro = ro; 1032 nd_region->numa_node = ndr_desc->numa_node; 1033 nd_region->target_node = ndr_desc->target_node; 1034 ida_init(&nd_region->ns_ida); 1035 ida_init(&nd_region->btt_ida); 1036 ida_init(&nd_region->pfn_ida); 1037 ida_init(&nd_region->dax_ida); 1038 dev = &nd_region->dev; 1039 dev_set_name(dev, "region%d", nd_region->id); 1040 dev->parent = &nvdimm_bus->dev; 1041 dev->type = dev_type; 1042 dev->groups = ndr_desc->attr_groups; 1043 dev->of_node = ndr_desc->of_node; 1044 nd_region->ndr_size = resource_size(ndr_desc->res); 1045 nd_region->ndr_start = ndr_desc->res->start; 1046 if (ndr_desc->flush) 1047 nd_region->flush = ndr_desc->flush; 1048 else 1049 nd_region->flush = NULL; 1050 1051 nd_device_register(dev); 1052 1053 return nd_region; 1054 1055 err_percpu: 1056 ida_simple_remove(®ion_ida, nd_region->id); 1057 err_id: 1058 kfree(region_buf); 1059 return NULL; 1060 } 1061 1062 struct nd_region *nvdimm_pmem_region_create(struct nvdimm_bus *nvdimm_bus, 1063 struct nd_region_desc *ndr_desc) 1064 { 1065 ndr_desc->num_lanes = ND_MAX_LANES; 1066 return nd_region_create(nvdimm_bus, ndr_desc, &nd_pmem_device_type, 1067 __func__); 1068 } 1069 EXPORT_SYMBOL_GPL(nvdimm_pmem_region_create); 1070 1071 struct nd_region *nvdimm_blk_region_create(struct nvdimm_bus *nvdimm_bus, 1072 struct nd_region_desc *ndr_desc) 1073 { 1074 if (ndr_desc->num_mappings > 1) 1075 return NULL; 1076 ndr_desc->num_lanes = min(ndr_desc->num_lanes, ND_MAX_LANES); 1077 return nd_region_create(nvdimm_bus, ndr_desc, &nd_blk_device_type, 1078 __func__); 1079 } 1080 EXPORT_SYMBOL_GPL(nvdimm_blk_region_create); 1081 1082 struct nd_region *nvdimm_volatile_region_create(struct nvdimm_bus *nvdimm_bus, 1083 struct nd_region_desc *ndr_desc) 1084 { 1085 ndr_desc->num_lanes = ND_MAX_LANES; 1086 return nd_region_create(nvdimm_bus, ndr_desc, &nd_volatile_device_type, 1087 __func__); 1088 } 1089 EXPORT_SYMBOL_GPL(nvdimm_volatile_region_create); 1090 1091 int nvdimm_flush(struct nd_region *nd_region, struct bio *bio) 1092 { 1093 int rc = 0; 1094 1095 if (!nd_region->flush) 1096 rc = generic_nvdimm_flush(nd_region); 1097 else { 1098 if (nd_region->flush(nd_region, bio)) 1099 rc = -EIO; 1100 } 1101 1102 return rc; 1103 } 1104 /** 1105 * nvdimm_flush - flush any posted write queues between the cpu and pmem media 1106 * @nd_region: blk or interleaved pmem region 1107 */ 1108 int generic_nvdimm_flush(struct nd_region *nd_region) 1109 { 1110 struct nd_region_data *ndrd = dev_get_drvdata(&nd_region->dev); 1111 int i, idx; 1112 1113 /* 1114 * Try to encourage some diversity in flush hint addresses 1115 * across cpus assuming a limited number of flush hints. 1116 */ 1117 idx = this_cpu_read(flush_idx); 1118 idx = this_cpu_add_return(flush_idx, hash_32(current->pid + idx, 8)); 1119 1120 /* 1121 * The first wmb() is needed to 'sfence' all previous writes 1122 * such that they are architecturally visible for the platform 1123 * buffer flush. Note that we've already arranged for pmem 1124 * writes to avoid the cache via memcpy_flushcache(). The final 1125 * wmb() ensures ordering for the NVDIMM flush write. 1126 */ 1127 wmb(); 1128 for (i = 0; i < nd_region->ndr_mappings; i++) 1129 if (ndrd_get_flush_wpq(ndrd, i, 0)) 1130 writeq(1, ndrd_get_flush_wpq(ndrd, i, idx)); 1131 wmb(); 1132 1133 return 0; 1134 } 1135 EXPORT_SYMBOL_GPL(nvdimm_flush); 1136 1137 /** 1138 * nvdimm_has_flush - determine write flushing requirements 1139 * @nd_region: blk or interleaved pmem region 1140 * 1141 * Returns 1 if writes require flushing 1142 * Returns 0 if writes do not require flushing 1143 * Returns -ENXIO if flushing capability can not be determined 1144 */ 1145 int nvdimm_has_flush(struct nd_region *nd_region) 1146 { 1147 int i; 1148 1149 /* no nvdimm or pmem api == flushing capability unknown */ 1150 if (nd_region->ndr_mappings == 0 1151 || !IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API)) 1152 return -ENXIO; 1153 1154 for (i = 0; i < nd_region->ndr_mappings; i++) { 1155 struct nd_mapping *nd_mapping = &nd_region->mapping[i]; 1156 struct nvdimm *nvdimm = nd_mapping->nvdimm; 1157 1158 /* flush hints present / available */ 1159 if (nvdimm->num_flush) 1160 return 1; 1161 } 1162 1163 /* 1164 * The platform defines dimm devices without hints, assume 1165 * platform persistence mechanism like ADR 1166 */ 1167 return 0; 1168 } 1169 EXPORT_SYMBOL_GPL(nvdimm_has_flush); 1170 1171 int nvdimm_has_cache(struct nd_region *nd_region) 1172 { 1173 return is_nd_pmem(&nd_region->dev) && 1174 !test_bit(ND_REGION_PERSIST_CACHE, &nd_region->flags); 1175 } 1176 EXPORT_SYMBOL_GPL(nvdimm_has_cache); 1177 1178 bool is_nvdimm_sync(struct nd_region *nd_region) 1179 { 1180 if (is_nd_volatile(&nd_region->dev)) 1181 return true; 1182 1183 return is_nd_pmem(&nd_region->dev) && 1184 !test_bit(ND_REGION_ASYNC, &nd_region->flags); 1185 } 1186 EXPORT_SYMBOL_GPL(is_nvdimm_sync); 1187 1188 struct conflict_context { 1189 struct nd_region *nd_region; 1190 resource_size_t start, size; 1191 }; 1192 1193 static int region_conflict(struct device *dev, void *data) 1194 { 1195 struct nd_region *nd_region; 1196 struct conflict_context *ctx = data; 1197 resource_size_t res_end, region_end, region_start; 1198 1199 if (!is_memory(dev)) 1200 return 0; 1201 1202 nd_region = to_nd_region(dev); 1203 if (nd_region == ctx->nd_region) 1204 return 0; 1205 1206 res_end = ctx->start + ctx->size; 1207 region_start = nd_region->ndr_start; 1208 region_end = region_start + nd_region->ndr_size; 1209 if (ctx->start >= region_start && ctx->start < region_end) 1210 return -EBUSY; 1211 if (res_end > region_start && res_end <= region_end) 1212 return -EBUSY; 1213 return 0; 1214 } 1215 1216 int nd_region_conflict(struct nd_region *nd_region, resource_size_t start, 1217 resource_size_t size) 1218 { 1219 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev); 1220 struct conflict_context ctx = { 1221 .nd_region = nd_region, 1222 .start = start, 1223 .size = size, 1224 }; 1225 1226 return device_for_each_child(&nvdimm_bus->dev, &ctx, region_conflict); 1227 } 1228 1229 void __exit nd_region_devs_exit(void) 1230 { 1231 ida_destroy(®ion_ida); 1232 } 1233